Implement --unstable-presymbolicate

fxprof-processed-profile/src/frame_table.rs

@@ -33,7 +33,7 @@ impl FrameTable {
         resource_table: &mut ResourceTable,
         func_table: &mut FuncTable,
         native_symbol_table: &mut NativeSymbols,
-        global_libs: &GlobalLibTable,
+        global_libs: &mut GlobalLibTable,
         frame: InternalFrame,
     ) -> usize {
         let addresses = &mut self.addresses;
@@ -73,6 +73,9 @@ impl FrameTable {
                                 (Some(native_symbol), name_string_index)
                             }
                             None => {
+                                // This isn't in the pre-provided symbol table, and we know it's in a library.
+                                global_libs.add_lib_used_rva(lib_index, address);
+
                                 let location_string = format!("0x{address:x}");
                                 (None, string_table.index_for_string(&location_string))
                             }

fxprof-processed-profile/src/global_lib_table.rs

@@ -1,3 +1,4 @@
+use std::collections::BTreeSet;
 use std::sync::Arc;
 
 use serde::ser::{Serialize, Serializer};
@@ -14,6 +15,11 @@ pub struct GlobalLibTable {
     used_libs: Vec<LibraryHandle>, // append-only for stable GlobalLibIndexes
     lib_map: FastHashMap<LibraryInfo, LibraryHandle>,
     used_lib_map: FastHashMap<LibraryHandle, GlobalLibIndex>,
+    /// We keep track of RVA addresses that exist in frames that are assigned to this
+    /// library, so that we can potentially provide symbolication info ahead of time.
+    /// This is here instead of in `LibraryInfo` because we don't want to serialize it,
+    /// and because it's currently a hack.
+    all_libs_seen_rvas: Vec<Option<BTreeSet<u32>>>,
 }
 
 impl GlobalLibTable {
@@ -23,6 +29,7 @@ impl GlobalLibTable {
             used_libs: Vec::new(),
             lib_map: FastHashMap::default(),
             used_lib_map: FastHashMap::default(),
+            all_libs_seen_rvas: Vec::new(),
         }
     }
 
@@ -31,6 +38,7 @@ impl GlobalLibTable {
         *self.lib_map.entry(lib.clone()).or_insert_with(|| {
             let handle = LibraryHandle(all_libs.len());
             all_libs.push(lib);
+            self.all_libs_seen_rvas.push(None);
             handle
         })
     }
@@ -52,6 +60,19 @@ impl GlobalLibTable {
         let handle = self.used_libs.get(index.0)?;
         self.all_libs.get(handle.0)
     }
+
+    pub fn add_lib_used_rva(&mut self, index: GlobalLibIndex, address: u32) {
+        let handle = self.used_libs.get(index.0).unwrap();
+        let lib_seen_rvas = self.all_libs_seen_rvas[handle.0].get_or_insert_with(BTreeSet::new);
+        lib_seen_rvas.insert(address);
+    }
+
+    pub fn lib_used_rva_iter(&self) -> UsedLibraryAddressesIterator {
+        UsedLibraryAddressesIterator {
+            next_used_lib_index: 0,
+            global_lib_table: self,
+        }
+    }
 }
 
 impl Serialize for GlobalLibTable {
@@ -73,4 +94,26 @@ impl Serialize for GlobalLibIndex {
 
 /// The handle for a library, obtained from [`Profile::add_lib`](crate::Profile::add_lib).
 #[derive(Debug, Clone, Copy, PartialOrd, Ord, PartialEq, Eq, Hash)]
-pub struct LibraryHandle(usize);
+pub struct LibraryHandle(pub(crate) usize);
+
+pub struct UsedLibraryAddressesIterator<'a> {
+    next_used_lib_index: usize,
+    global_lib_table: &'a GlobalLibTable,
+}
+
+impl<'a> Iterator for UsedLibraryAddressesIterator<'a> {
+    type Item = (&'a LibraryInfo, Option<&'a BTreeSet<u32>>);
+
+    fn next(&mut self) -> Option<Self::Item> {
+        self.global_lib_table
+            .used_libs
+            .get(self.next_used_lib_index)
+            .map(|lib| {
+                self.next_used_lib_index += 1;
+                (
+                    &self.global_lib_table.all_libs[lib.0],
+                    self.global_lib_table.all_libs_seen_rvas[lib.0].as_ref(),
+                )
+            })
+    }
+}

fxprof-processed-profile/src/lib.rs

@@ -63,7 +63,7 @@ pub use category_color::CategoryColor;
 pub use counters::CounterHandle;
 pub use cpu_delta::CpuDelta;
 pub use frame::{Frame, FrameFlags, FrameInfo};
-pub use global_lib_table::LibraryHandle;
+pub use global_lib_table::{LibraryHandle, UsedLibraryAddressesIterator};
 pub use lib_mappings::LibMappings;
 pub use library_info::{LibraryInfo, Symbol, SymbolTable};
 pub use markers::*;

fxprof-processed-profile/src/profile.rs

@@ -11,7 +11,7 @@ use crate::cpu_delta::CpuDelta;
 use crate::fast_hash_map::FastHashMap;
 use crate::frame::{Frame, FrameInfo};
 use crate::frame_table::{InternalFrame, InternalFrameLocation};
-use crate::global_lib_table::{GlobalLibTable, LibraryHandle};
+use crate::global_lib_table::{GlobalLibTable, LibraryHandle, UsedLibraryAddressesIterator};
 use crate::lib_mappings::LibMappings;
 use crate::library_info::LibraryInfo;
 use crate::process::{Process, ThreadHandle};
@@ -601,7 +601,7 @@ impl Profile {
                 flags: frame_info.flags,
                 category_pair: frame_info.category_pair,
             };
-            let frame_index = thread.frame_index_for_frame(internal_frame, &self.global_libs);
+            let frame_index = thread.frame_index_for_frame(internal_frame, &mut self.global_libs);
             prefix =
                 Some(thread.stack_index_for_stack(prefix, frame_index, frame_info.category_pair));
         }
@@ -665,6 +665,10 @@ impl Profile {
     fn contains_js_function(&self) -> bool {
         self.threads.iter().any(|t| t.contains_js_function())
     }
+
+    pub fn lib_used_rva_iter(&self) -> UsedLibraryAddressesIterator {
+        self.global_libs.lib_used_rva_iter()
+    }
 }
 
 impl Serialize for Profile {

fxprof-processed-profile/src/thread.rs

@@ -97,7 +97,7 @@ impl Thread {
     pub fn frame_index_for_frame(
         &mut self,
         frame: InternalFrame,
-        global_libs: &GlobalLibTable,
+        global_libs: &mut GlobalLibTable,
     ) -> usize {
         self.frame_table.index_for_frame(
             &mut self.string_table,

samply-symbols/src/lib.rs

@@ -256,7 +256,7 @@ pub use crate::shared::{
     MultiArchDisambiguator, OptionallySendFuture, PeCodeId, SourceFilePath, SymbolInfo,
     SyncAddressInfo,
 };
-pub use crate::symbol_map::SymbolMap;
+pub use crate::symbol_map::{SymbolMap, SymbolMapTrait};
 
 pub struct SymbolManager<H: FileAndPathHelper> {
     helper: Arc<H>,
@@ -304,6 +304,14 @@ where
     /// Obtain a symbol map for the library, given the (partial) `LibraryInfo`.
     /// At least the debug_id has to be given.
     pub async fn load_symbol_map(&self, library_info: &LibraryInfo) -> Result<SymbolMap<H>, Error> {
+        if let Some((fl, symbol_map)) = self
+            .helper()
+            .as_ref()
+            .get_symbol_map_for_library(library_info)
+        {
+            return Ok(SymbolMap::with_symbol_map_trait(fl, symbol_map));
+        }
+
         let debug_id = match library_info.debug_id {
             Some(debug_id) => debug_id,
             None => return Err(Error::NotEnoughInformationToIdentifySymbolMap),

samply-symbols/src/shared.rs

@@ -5,6 +5,7 @@ use std::future::Future;
 use std::marker::PhantomData;
 use std::ops::{Deref, Range};
 use std::str::FromStr;
+use std::sync::Arc;
 
 #[cfg(feature = "partial_read_stats")]
 use bitvec::{bitvec, prelude::BitVec};
@@ -14,6 +15,7 @@ use object::FileFlags;
 use uuid::Uuid;
 
 use crate::mapped_path::MappedPath;
+use crate::symbol_map::SymbolMapTrait;
 
 pub type FileAndPathHelperError = Box<dyn std::error::Error + Send + Sync + 'static>;
 pub type FileAndPathHelperResult<T> = std::result::Result<T, FileAndPathHelperError>;
@@ -402,6 +404,14 @@ pub trait FileAndPathHelper {
         &self,
         location: Self::FL,
     ) -> std::pin::Pin<Box<dyn OptionallySendFuture<Output = FileAndPathHelperResult<Self::F>> + '_>>;
+
+    /// Ask the helper to return a SymbolMap if it happens to have one available already.
+    fn get_symbol_map_for_library(
+        &self,
+        _info: &LibraryInfo,
+    ) -> Option<(Self::FL, Arc<dyn SymbolMapTrait + Send + Sync>)> {
+        None
+    }
 }
 
 /// Provides synchronous access to the raw bytes of a file.

samply-symbols/src/symbol_map.rs

@@ -42,6 +42,7 @@ pub trait GetInnerSymbolMapWithLookupFramesExt<FC> {
 enum InnerSymbolMap<FC> {
     WithoutAddFile(Box<dyn GetInnerSymbolMap + Send + Sync>),
     WithAddFile(Box<dyn GetInnerSymbolMapWithLookupFramesExt<FC> + Send + Sync>),
+    Direct(Arc<dyn SymbolMapTrait + Send + Sync>),
 }
 
 pub struct SymbolMap<H: FileAndPathHelper> {
@@ -74,10 +75,22 @@ impl<H: FileAndPathHelper> SymbolMap<H> {
         }
     }
 
+    pub fn with_symbol_map_trait(
+        debug_file_location: H::FL,
+        inner: Arc<dyn SymbolMapTrait + Send + Sync>,
+    ) -> Self {
+        Self {
+            debug_file_location,
+            inner: InnerSymbolMap::Direct(inner),
+            helper: None,
+        }
+    }
+
     fn inner(&self) -> &dyn SymbolMapTrait {
         match &self.inner {
             InnerSymbolMap::WithoutAddFile(inner) => inner.get_inner_symbol_map(),
             InnerSymbolMap::WithAddFile(inner) => inner.get_inner_symbol_map().get_as_symbol_map(),
+            InnerSymbolMap::Direct(inner) => inner.as_ref(),
         }
     }
 
@@ -111,7 +124,7 @@ impl<H: FileAndPathHelper> SymbolMap<H> {
                     frames: Some(frames),
                 });
             }
-            (None, _) | (_, InnerSymbolMap::WithoutAddFile(_)) => {
+            (None, _) | (_, InnerSymbolMap::WithoutAddFile(_)) | (_, InnerSymbolMap::Direct(_)) => {
                 return Some(AddressInfo {
                     symbol,
                     frames: None,
@@ -168,7 +181,7 @@ impl<H: FileAndPathHelper> SymbolMap<H> {
     ) -> Option<Vec<FrameDebugInfo>> {
         let helper = self.helper.as_deref()?;
         let inner = match &self.inner {
-            InnerSymbolMap::WithoutAddFile(_) => return None,
+            InnerSymbolMap::WithoutAddFile(_) | InnerSymbolMap::Direct(_) => return None,
             InnerSymbolMap::WithAddFile(inner) => inner.get_inner_symbol_map(),
         };
 

samply/src/linux/profiler.rs

@@ -89,6 +89,7 @@ pub fn start_recording(
     let interval = recording_props.interval;
     let time_limit = recording_props.time_limit;
     let observer_thread = thread::spawn(move || {
+        let unstable_presymbolicate = profile_creation_props.unstable_presymbolicate;
         let mut converter = make_converter(interval, profile_creation_props);
 
         // Wait for the initial pid to profile.
@@ -120,6 +121,7 @@ pub fn start_recording(
             profile_another_pid_request_receiver,
             profile_another_pid_reply_sender,
             stop_receiver,
+            unstable_presymbolicate,
         );
     });
 
@@ -257,6 +259,7 @@ fn start_profiling_pid(
         move || {
             let interval = recording_props.interval;
             let time_limit = recording_props.time_limit;
+            let unstable_presymbolicate = profile_creation_props.unstable_presymbolicate;
             let mut converter = make_converter(interval, profile_creation_props);
             let SamplerRequest::StartProfilingAnotherProcess(pid, attach_mode) =
                 profile_another_pid_request_receiver.recv().unwrap()
@@ -277,6 +280,7 @@ fn start_profiling_pid(
                 profile_another_pid_request_receiver,
                 profile_another_pid_reply_sender,
                 ctrl_c_receiver,
+                unstable_presymbolicate,
             )
         }
     });
@@ -524,6 +528,7 @@ enum SamplerRequest {
     StopProfilingOncePerfEventsExhausted,
 }
 
+#[allow(clippy::too_many_arguments)]
 fn run_profiler(
     mut perf: PerfGroup,
     mut converter: Converter<
@@ -534,6 +539,7 @@ fn run_profiler(
     more_processes_request_receiver: Receiver<SamplerRequest>,
     more_processes_reply_sender: Sender<bool>,
     mut stop_receiver: oneshot::Receiver<()>,
+    unstable_presymbolicate: bool,
 ) {
     // eprintln!("Running...");
 
@@ -659,9 +665,18 @@ fn run_profiler(
 
     let profile = converter.finish();
 
-    let output_file = File::create(output_filename).unwrap();
-    let writer = BufWriter::new(output_file);
-    serde_json::to_writer(writer, &profile).expect("Couldn't write JSON");
+    {
+        let output_file = File::create(output_filename).unwrap();
+        let writer = BufWriter::new(output_file);
+        serde_json::to_writer(writer, &profile).expect("Couldn't write JSON");
+    }
+
+    if unstable_presymbolicate {
+        crate::shared::symbol_precog::presymbolicate(
+            &profile,
+            &output_filename.with_extension("syms.json"),
+        );
+    }
 }
 
 pub fn read_string_lossy<P: AsRef<Path>>(path: P) -> std::io::Result<String> {

samply/src/mac/profiler.rs

@@ -85,6 +85,8 @@ pub fn start_recording(
         ..profile_creation_props
     };
 
+    let unstable_presymbolicate = profile_creation_props.unstable_presymbolicate;
+
     let (task_sender, task_receiver) = unbounded();
 
     let sampler_thread = thread::spawn(move || {
@@ -210,6 +212,13 @@ pub fn start_recording(
         to_writer(writer, &profile).expect("Couldn't write JSON");
     }
 
+    if unstable_presymbolicate {
+        crate::shared::symbol_precog::presymbolicate(
+            &profile,
+            &output_file.with_extension("syms.json"),
+        );
+    }
+
     if let Some(server_props) = server_props {
         let libinfo_map = crate::profile_json_preparse::parse_libinfo_map_from_profile_file(
             File::open(&output_file).expect("Couldn't open file we just wrote"),

samply/src/main.rs

@@ -237,6 +237,14 @@ pub struct ProfileCreationArgs {
     /// Create a separate thread for each CPU. Not supported on macOS
     #[arg(long)]
     per_cpu_threads: bool,
+
+    /// Emit .syms.json sidecar file containing gathered symbol info for all frames referenced by
+    /// this profile. With this file along with the profile, samply can load the profile
+    /// and provide symbols to the front end without needing debug files to be
+    /// available. (Unstable: will probably change to include the full information
+    /// in the profile.json, instead of a sidecar file.)
+    #[arg(long)]
+    unstable_presymbolicate: bool,
 }
 
 #[derive(Debug, Args)]
@@ -369,6 +377,7 @@ impl ImportArgs {
             unlink_aux_files: self.profile_creation_args.unlink_aux_files,
             create_per_cpu_threads: self.profile_creation_args.per_cpu_threads,
             override_arch: self.override_arch.clone(),
+            unstable_presymbolicate: self.profile_creation_args.unstable_presymbolicate,
         }
     }
 
@@ -472,6 +481,7 @@ impl RecordArgs {
             unlink_aux_files: self.profile_creation_args.unlink_aux_files,
             create_per_cpu_threads: self.profile_creation_args.per_cpu_threads,
             override_arch: None,
+            unstable_presymbolicate: self.profile_creation_args.unstable_presymbolicate,
         }
     }
 }