diff --git a/Cargo.lock b/Cargo.lock index 60d57c154d..1e92c7fa39 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -32,6 +32,16 @@ version = "1.0.71" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c7d0618f0e0b7e8ff11427422b64564d5fb0be1940354bfe2e0529b18a9d9b8" +[[package]] +name = "app_units" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08048ef9fa8600604791786712c3a9bad1038415b91845761ff1b3305b1ed187" +dependencies = [ + "num-traits", + "serde", +] + [[package]] name = "arrayref" version = "0.3.6" @@ -61,7 +71,7 @@ dependencies = [ "proc-macro2", "quote", "serde", - "syn", + "syn 2.0.26", ] [[package]] @@ -213,7 +223,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1f34ba9a9bcb8645379e9de8cb3ecfcf4d1c85ba66d90deb3259206fa5aa193b" dependencies = [ "quote", - "syn", + "syn 2.0.26", ] [[package]] @@ -273,6 +283,15 @@ dependencies = [ "libc", ] +[[package]] +name = "euclid" +version = "0.20.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bb7ef65b3777a325d1eeefefab5b6d4959da54747e33bd6258e789640f307ad" +dependencies = [ + "num-traits", +] + [[package]] name = "fastrand" version = "2.0.0" @@ -342,6 +361,7 @@ dependencies = [ "serde_json", "tempfile", "whatsys", + "wr_malloc_size_of", ] [[package]] @@ -379,6 +399,7 @@ dependencies = [ "flate2", "iso8601", "log", + "malloc_size_of_derive", "once_cell", "oslog", "rkv", @@ -389,6 +410,7 @@ dependencies = [ "time", "uniffi", "uuid", + "wr_malloc_size_of", "zeitstempel", ] @@ -454,7 +476,7 @@ checksum = "b311f3b85c4fe018ce74d962fec93365819da2219e1d6529d66db241e7c70ce0" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.26", ] [[package]] @@ -561,6 +583,17 @@ version = "0.4.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" +[[package]] +name = "malloc_size_of_derive" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "632647502a8bfa82458c07134791fffa7a719f00427d1afd79c3cb6d4960a982" +dependencies = [ + "proc-macro2", + "syn 1.0.109", + "synstructure", +] + [[package]] name = "memchr" version = "2.5.0" @@ -805,7 +838,7 @@ checksum = "7f81c2fde025af7e69b1d1420531c8a8811ca898919db177141a85313b1cb932" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.26", ] [[package]] @@ -834,7 +867,7 @@ checksum = "741e124f5485c7e60c03b043f79f320bff3527f4bbf12cf3831750dc46a0ec2c" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.26", ] [[package]] @@ -866,6 +899,17 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "syn" version = "2.0.26" @@ -877,6 +921,18 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "synstructure" +version = "0.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f36bdaa60a83aca3921b5259d5400cbf5e90fc51931376a9bd4a0eb79aa7210f" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", + "unicode-xid", +] + [[package]] name = "tempfile" version = "3.8.0" @@ -918,7 +974,7 @@ checksum = "f9456a42c5b0d803c8cd86e73dd7cc9edd429499f37a3550d286d5e86720569f" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.26", ] [[package]] @@ -998,6 +1054,12 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e51733f11c9c4f72aa0c160008246859e340b00807569a0da0e7a1079b27ba85" +[[package]] +name = "unicode-xid" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f962df74c8c05a667b5ee8bcf162993134c104e96440b663c8daa176dc772d8c" + [[package]] name = "uniffi" version = "0.27.0" @@ -1053,7 +1115,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1f7f91c2de378a5993a6d0367d9c6e178bbc98309919ee42fccc0142a5adbb25" dependencies = [ "quote", - "syn", + "syn 2.0.26", ] [[package]] @@ -1085,7 +1147,7 @@ dependencies = [ "proc-macro2", "quote", "serde", - "syn", + "syn 2.0.26", "toml", "uniffi_meta", ] @@ -1274,6 +1336,16 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" +[[package]] +name = "wr_malloc_size_of" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db83ceee1d8ac1591e1b053eba4580f58009155b5d8acfd66c1cfd369e593c89" +dependencies = [ + "app_units", + "euclid", +] + [[package]] name = "xshell" version = "0.2.2" diff --git a/glean-core/Cargo.toml b/glean-core/Cargo.toml index 611d0e3df2..bff84b2af9 100644 --- a/glean-core/Cargo.toml +++ b/glean-core/Cargo.toml @@ -43,6 +43,10 @@ thiserror = "1.0.4" uniffi = { version = "0.27.0", default-features = false } time = "0.1.40" env_logger = { version = "0.10.0", default-features = false, optional = true } +# the version on crates.io depends on syn v1, but m-c uses a patched on with syn v2 +malloc_size_of_derive = "0.1.2" +# crates.io has 0.1.0, m-c has 0.0.2, this makes things highly confusing. +malloc_size_of = { version = "< 1", package = "wr_malloc_size_of" } [target.'cfg(target_os = "android")'.dependencies] android_logger = { version = "0.12.0", default-features = false } diff --git a/glean-core/rlb/Cargo.toml b/glean-core/rlb/Cargo.toml index 113de95e23..47102a31ef 100644 --- a/glean-core/rlb/Cargo.toml +++ b/glean-core/rlb/Cargo.toml @@ -30,6 +30,8 @@ inherent = "1" log = "0.4.8" once_cell = "1.18.0" whatsys = "0.3.0" +# crates.io has 0.1.0, m-c has 0.0.2, this makes things highly confusing. +malloc_size_of = { version = "< 1", package = "wr_malloc_size_of" } [dev-dependencies] crossbeam-channel = "0.5" diff --git a/glean-core/rlb/src/private/ping.rs b/glean-core/rlb/src/private/ping.rs index 6c126992bc..c389794679 100644 --- a/glean-core/rlb/src/private/ping.rs +++ b/glean-core/rlb/src/private/ping.rs @@ -2,7 +2,12 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. -use std::sync::{Arc, Mutex}; +use std::{ + mem, + sync::{Arc, Mutex}, +}; + +use malloc_size_of::MallocSizeOf; type BoxedCallback = Box) + Send + 'static>; @@ -19,6 +24,19 @@ pub struct PingType { test_callback: Arc>>, } +impl MallocSizeOf for PingType { + fn size_of(&self, ops: &mut malloc_size_of::MallocSizeOfOps) -> usize { + self.inner.size_of(ops) + + self + .test_callback + .lock() + .unwrap() + .as_ref() + .map(|cb| mem::size_of_val(cb)) + .unwrap_or(0) + } +} + impl PingType { /// Creates a new ping type. /// diff --git a/glean-core/src/common_metric_data.rs b/glean-core/src/common_metric_data.rs index f5058d995f..941a8cd2a7 100644 --- a/glean-core/src/common_metric_data.rs +++ b/glean-core/src/common_metric_data.rs @@ -4,6 +4,8 @@ use std::sync::atomic::{AtomicU8, Ordering}; +use malloc_size_of_derive::MallocSizeOf; + use crate::error::{Error, ErrorKind}; use crate::metrics::labeled::validate_dynamic_label; use crate::Glean; @@ -12,7 +14,7 @@ use serde::{Deserialize, Serialize}; /// The supported metrics' lifetimes. /// /// A metric's lifetime determines when its stored data gets reset. -#[derive(Copy, Clone, Debug, PartialEq, Eq, Deserialize, Serialize, Default)] +#[derive(Copy, Clone, Debug, PartialEq, Eq, Deserialize, Serialize, Default, MallocSizeOf)] #[repr(i32)] // Use i32 to be compatible with our JNA definition #[serde(rename_all = "lowercase")] pub enum Lifetime { @@ -50,7 +52,7 @@ impl TryFrom for Lifetime { } /// The common set of data shared across all different metric types. -#[derive(Default, Debug, Clone, Deserialize, Serialize)] +#[derive(Default, Debug, Clone, Deserialize, Serialize, MallocSizeOf)] pub struct CommonMetricData { /// The metric's name. pub name: String, @@ -73,9 +75,10 @@ pub struct CommonMetricData { pub dynamic_label: Option, } -#[derive(Default, Debug)] +#[derive(Default, Debug, MallocSizeOf)] pub struct CommonMetricDataInternal { pub inner: CommonMetricData, + #[ignore_malloc_size_of = "atomic not allocated"] pub disabled: AtomicU8, } diff --git a/glean-core/src/core/mod.rs b/glean-core/src/core/mod.rs index f69f0c3868..f4160a63d4 100644 --- a/glean-core/src/core/mod.rs +++ b/glean-core/src/core/mod.rs @@ -8,6 +8,7 @@ use std::sync::atomic::{AtomicU8, Ordering}; use std::sync::{Arc, Mutex}; use chrono::{DateTime, FixedOffset}; +use malloc_size_of_derive::MallocSizeOf; use once_cell::sync::OnceCell; use crate::database::Database; @@ -142,7 +143,7 @@ where /// /// In specific language bindings, this is usually wrapped in a singleton and all metric recording goes to a single instance of this object. /// In the Rust core, it is possible to create multiple instances, which is used in testing. -#[derive(Debug)] +#[derive(Debug, MallocSizeOf)] pub struct Glean { upload_enabled: bool, pub(crate) data_store: Option, @@ -151,9 +152,11 @@ pub struct Glean { pub(crate) additional_metrics: AdditionalMetrics, pub(crate) database_metrics: DatabaseMetrics, pub(crate) internal_pings: InternalPings, + #[ignore_malloc_size_of = "libstd type"] data_path: PathBuf, application_id: String, ping_registry: HashMap, + #[ignore_malloc_size_of = "non-allocating type"] start_time: DateTime, max_events: u32, is_first_run: bool, @@ -161,7 +164,9 @@ pub struct Glean { debug: DebugOptions, pub(crate) app_build: String, pub(crate) schedule_metrics_pings: bool, + #[ignore_malloc_size_of = "atomic not allocated"] pub(crate) remote_settings_epoch: AtomicU8, + #[ignore_malloc_size_of = "TODO"] pub(crate) remote_settings_metrics_config: Arc>, pub(crate) with_timestamps: bool, } diff --git a/glean-core/src/core_metrics.rs b/glean-core/src/core_metrics.rs index baa2b8515b..4757e5712b 100644 --- a/glean-core/src/core_metrics.rs +++ b/glean-core/src/core_metrics.rs @@ -7,16 +7,18 @@ use crate::metrics::{ }; use crate::{CommonMetricData, Lifetime}; +use malloc_size_of_derive::MallocSizeOf; use once_cell::sync::Lazy; /// Metrics included in every ping as `client_info`. -#[derive(Debug, Default)] +#[derive(Debug, Default, MallocSizeOf)] pub struct ClientInfoMetrics { /// The build identifier generated by the CI system (e.g. "1234/A"). pub app_build: String, /// The user visible version string (e.g. "1.0.3"). pub app_display_version: String, /// The app's build date + #[ignore_malloc_size_of = "not an allocated value"] pub app_build_date: Datetime, /// The architecture of the device (e.g. "arm", "x86"). diff --git a/glean-core/src/database/mod.rs b/glean-core/src/database/mod.rs index 0dbf0220bc..0cd0c5fc2f 100644 --- a/glean-core/src/database/mod.rs +++ b/glean-core/src/database/mod.rs @@ -13,6 +13,7 @@ use std::sync::RwLock; use crate::ErrorKind; +use malloc_size_of::MallocSizeOf; use rkv::migrator::Migrator; use rkv::{MigrateError, StoreError, StoreOptions}; @@ -191,6 +192,30 @@ pub struct Database { rkv_load_state: RkvLoadState, } +impl MallocSizeOf for Database { + fn size_of(&self, _ops: &mut malloc_size_of::MallocSizeOfOps) -> usize { + // TODO: Fill in gaps. + + let mut n = 0; + + n += 0; // self.rkv.size_of(ops) -- not implemented. + n += 0; // self.user_store.size_of(ops) -- not implemented. + + n += self + .ping_lifetime_data + .as_ref() + .map(|_data| { + // TODO: servo's malloc_size_of implements it for BTreeMap. + //let lock = data.read().unwrap(); + //(*lock).size_of(ops) + 0 + }) + .unwrap_or(0); + + n + } +} + impl std::fmt::Debug for Database { fn fmt(&self, fmt: &mut std::fmt::Formatter) -> std::fmt::Result { fmt.debug_struct("Database") diff --git a/glean-core/src/debug.rs b/glean-core/src/debug.rs index 88f807bd88..1f1342b020 100644 --- a/glean-core/src/debug.rs +++ b/glean-core/src/debug.rs @@ -27,12 +27,16 @@ use std::env; +use malloc_size_of::MallocSizeOf; +use malloc_size_of_derive::MallocSizeOf; + const GLEAN_LOG_PINGS: &str = "GLEAN_LOG_PINGS"; const GLEAN_DEBUG_VIEW_TAG: &str = "GLEAN_DEBUG_VIEW_TAG"; const GLEAN_SOURCE_TAGS: &str = "GLEAN_SOURCE_TAGS"; const GLEAN_MAX_SOURCE_TAGS: usize = 5; /// A representation of all of Glean's debug options. +#[derive(MallocSizeOf)] pub struct DebugOptions { /// Option to log the payload of pings that are successfully assembled into a ping request. pub log_pings: DebugOption, @@ -83,6 +87,15 @@ pub struct DebugOption Option, V = fn(&T) -> bool> { validation: Option, } +impl MallocSizeOf for DebugOption +where + T: MallocSizeOf, +{ + fn size_of(&self, ops: &mut malloc_size_of::MallocSizeOfOps) -> usize { + self.env.size_of(ops) + self.value.size_of(ops) + } +} + impl DebugOption where T: Clone, diff --git a/glean-core/src/event_database/mod.rs b/glean-core/src/event_database/mod.rs index 50b2488a4c..445006b190 100644 --- a/glean-core/src/event_database/mod.rs +++ b/glean-core/src/event_database/mod.rs @@ -14,6 +14,7 @@ use std::sync::RwLock; use chrono::{DateTime, FixedOffset, Utc}; +use malloc_size_of::MallocSizeOf; use serde::{Deserialize, Serialize}; use serde_json::{json, Value as JsonValue}; @@ -28,7 +29,9 @@ use crate::Result; use crate::{CommonMetricData, CounterMetric, Lifetime}; /// Represents the recorded data for a single event. -#[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq)] +#[derive( + Debug, Clone, Deserialize, Serialize, PartialEq, Eq, malloc_size_of_derive::MallocSizeOf, +)] #[cfg_attr(test, derive(Default))] pub struct RecordedEvent { /// The timestamp of when the event was recorded. @@ -54,7 +57,9 @@ pub struct RecordedEvent { } /// Represents the stored data for a single event. -#[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq)] +#[derive( + Debug, Clone, Deserialize, Serialize, PartialEq, Eq, malloc_size_of_derive::MallocSizeOf, +)] struct StoredEvent { #[serde(flatten)] event: RecordedEvent, @@ -99,6 +104,12 @@ pub struct EventDatabase { file_lock: RwLock<()>, } +impl MallocSizeOf for EventDatabase { + fn size_of(&self, ops: &mut malloc_size_of::MallocSizeOfOps) -> usize { + self.event_stores.read().unwrap().size_of(ops) + } +} + impl EventDatabase { /// Creates a new event database. /// diff --git a/glean-core/src/histogram/exponential.rs b/glean-core/src/histogram/exponential.rs index 5481c4feb9..9cd5e6a915 100644 --- a/glean-core/src/histogram/exponential.rs +++ b/glean-core/src/histogram/exponential.rs @@ -4,6 +4,7 @@ use std::collections::HashMap; +use malloc_size_of_derive::MallocSizeOf; use once_cell::sync::OnceCell; use serde::{Deserialize, Serialize}; @@ -56,11 +57,12 @@ fn exponential_range(min: u64, max: u64, bucket_count: usize) -> Vec { /// /// Buckets are pre-computed at instantiation with an exponential distribution from `min` to `max` /// and `bucket_count` buckets. -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, MallocSizeOf)] pub struct PrecomputedExponential { // Don't serialize the (potentially large) array of ranges, instead compute them on first // access. #[serde(skip)] + #[ignore_malloc_size_of = "TODO"] bucket_ranges: OnceCell>, min: u64, max: u64, diff --git a/glean-core/src/histogram/functional.rs b/glean-core/src/histogram/functional.rs index 64df9a1a4d..4ab9e8de03 100644 --- a/glean-core/src/histogram/functional.rs +++ b/glean-core/src/histogram/functional.rs @@ -4,6 +4,7 @@ use std::collections::HashMap; +use malloc_size_of_derive::MallocSizeOf; use serde::{Deserialize, Serialize}; use super::{Bucketing, Histogram}; @@ -18,7 +19,7 @@ use crate::util::floating_point_context::FloatingPointContext; /// i = ⌊n logbase(𝑥)⌋ /// /// In other words, there are n buckets for each power of `base` magnitude. -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, MallocSizeOf)] pub struct Functional { exponent: f64, } diff --git a/glean-core/src/histogram/linear.rs b/glean-core/src/histogram/linear.rs index 7b30ea8f6c..b1d41cec13 100644 --- a/glean-core/src/histogram/linear.rs +++ b/glean-core/src/histogram/linear.rs @@ -5,6 +5,7 @@ use std::cmp; use std::collections::HashMap; +use malloc_size_of_derive::MallocSizeOf; use once_cell::sync::OnceCell; use serde::{Deserialize, Serialize}; @@ -36,11 +37,12 @@ fn linear_range(min: u64, max: u64, count: usize) -> Vec { /// /// Buckets are pre-computed at instantiation with a linear distribution from `min` to `max` /// and `bucket_count` buckets. -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, MallocSizeOf)] pub struct PrecomputedLinear { // Don't serialize the (potentially large) array of ranges, instead compute them on first // access. #[serde(skip)] + #[ignore_malloc_size_of = "TODO"] bucket_ranges: OnceCell>, min: u64, max: u64, diff --git a/glean-core/src/histogram/mod.rs b/glean-core/src/histogram/mod.rs index 6e2880dffa..61ce6be7b3 100644 --- a/glean-core/src/histogram/mod.rs +++ b/glean-core/src/histogram/mod.rs @@ -6,6 +6,7 @@ use std::collections::HashMap; +use malloc_size_of_derive::MallocSizeOf; use serde::{Deserialize, Serialize}; use crate::error::{Error, ErrorKind}; @@ -19,7 +20,7 @@ mod functional; mod linear; /// Different kinds of histograms. -#[derive(Debug, Clone, Copy, Serialize, Deserialize)] +#[derive(Debug, Clone, Copy, Serialize, Deserialize, MallocSizeOf)] #[serde(rename_all = "lowercase")] pub enum HistogramType { /// A histogram with linear distributed buckets. @@ -57,7 +58,7 @@ impl TryFrom for HistogramType { /// assert_eq!(10, hist.count()); /// assert_eq!(55, hist.sum()); /// ``` -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, MallocSizeOf)] pub struct Histogram { /// Mapping bucket's minimum to sample count. values: HashMap, diff --git a/glean-core/src/internal_metrics.rs b/glean-core/src/internal_metrics.rs index 3703417466..51c5d9050f 100644 --- a/glean-core/src/internal_metrics.rs +++ b/glean-core/src/internal_metrics.rs @@ -4,16 +4,18 @@ use std::borrow::Cow; +use malloc_size_of_derive::MallocSizeOf; + use super::{metrics::*, CommonMetricData, Lifetime}; -#[derive(Debug)] +#[derive(Debug, MallocSizeOf)] pub struct CoreMetrics { pub client_id: UuidMetric, pub first_run_date: DatetimeMetric, pub os: StringMetric, } -#[derive(Debug)] +#[derive(Debug, MallocSizeOf)] pub struct AdditionalMetrics { /// The number of times we encountered an IO error /// when writing a pending ping to disk. @@ -137,7 +139,7 @@ impl AdditionalMetrics { } } -#[derive(Debug)] +#[derive(Debug, MallocSizeOf)] pub struct UploadMetrics { pub ping_upload_failure: LabeledMetric, pub discarded_exceeding_pings_size: MemoryDistributionMetric, @@ -258,7 +260,7 @@ impl UploadMetrics { } } -#[derive(Debug)] +#[derive(Debug, MallocSizeOf)] pub struct DatabaseMetrics { pub size: MemoryDistributionMetric, diff --git a/glean-core/src/internal_pings.rs b/glean-core/src/internal_pings.rs index 1cf32feb60..5d3ca3fd24 100644 --- a/glean-core/src/internal_pings.rs +++ b/glean-core/src/internal_pings.rs @@ -2,6 +2,8 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. +use malloc_size_of_derive::MallocSizeOf; + use crate::metrics::PingType; /// Glean-provided pings, all enabled by default. @@ -10,7 +12,7 @@ use crate::metrics::PingType; /// This might get auto-generated when the Rust API lands ([Bug 1579146](https://bugzilla.mozilla.org/show_bug.cgi?id=1579146)). /// /// They are parsed and registered by the platform-specific wrappers, but might be used Glean-internal directly. -#[derive(Debug, Clone)] +#[derive(Debug, Clone, MallocSizeOf)] pub struct InternalPings { pub baseline: PingType, pub metrics: PingType, diff --git a/glean-core/src/lib.rs b/glean-core/src/lib.rs index af68fde264..fa512cd1bb 100644 --- a/glean-core/src/lib.rs +++ b/glean-core/src/lib.rs @@ -25,6 +25,7 @@ use std::time::Duration; use crossbeam_channel::unbounded; use log::LevelFilter; +use malloc_size_of_derive::MallocSizeOf; use once_cell::sync::{Lazy, OnceCell}; use uuid::Uuid; @@ -104,7 +105,7 @@ static INIT_HANDLES: Lazy>>>> = Lazy::new(|| Arc::new(Mutex::new(Vec::new()))); /// Configuration for Glean -#[derive(Debug, Clone)] +#[derive(Debug, Clone, MallocSizeOf)] pub struct InternalConfiguration { /// Whether upload should be enabled. pub upload_enabled: bool, @@ -126,6 +127,8 @@ pub struct InternalConfiguration { /// Whether Glean should, on init, trim its event storage to only the registered pings. pub trim_data_to_registered_pings: bool, /// The internal logging level. + /// ignore + #[ignore_malloc_size_of = "external type"] pub log_level: Option, /// The rate at which pings may be uploaded before they are throttled. pub rate_limit: Option, @@ -140,7 +143,7 @@ pub struct InternalConfiguration { } /// How to specify the rate at which pings may be uploaded before they are throttled. -#[derive(Debug, Clone)] +#[derive(Debug, Clone, MallocSizeOf)] pub struct PingRateLimit { /// Length of time in seconds of a ping uploading interval. pub seconds_per_interval: u64, diff --git a/glean-core/src/metrics/labeled.rs b/glean-core/src/metrics/labeled.rs index f9f6a28880..da5e456ca6 100644 --- a/glean-core/src/metrics/labeled.rs +++ b/glean-core/src/metrics/labeled.rs @@ -4,8 +4,11 @@ use std::borrow::Cow; use std::collections::{hash_map::Entry, HashMap}; +use std::mem; use std::sync::{Arc, Mutex}; +use malloc_size_of::MallocSizeOf; + use crate::common_metric_data::{CommonMetricData, CommonMetricDataInternal}; use crate::error_recording::{record_error, test_get_num_recorded_errors, ErrorType}; use crate::metrics::{BooleanMetric, CounterMetric, Metric, MetricType, StringMetric}; @@ -39,6 +42,32 @@ pub struct LabeledMetric { label_map: Mutex>>, } +impl ::malloc_size_of::MallocSizeOf for LabeledMetric { + fn size_of(&self, ops: &mut malloc_size_of::MallocSizeOfOps) -> usize { + let map = self.label_map.lock().unwrap(); + + // Copy of `MallocShallowSizeOf` implementation for `HashMap`. + // Note: An instantiated submetric is behind an `Arc`. + // `size_of` should only be called on the main thread to avoid double-counting. + let shallow_size = if ops.has_malloc_enclosing_size_of() { + map.values() + .next() + .map_or(0, |v| unsafe { ops.malloc_enclosing_size_of(v) }) + } else { + map.capacity() + * (mem::size_of::() + mem::size_of::() + mem::size_of::()) + }; + + let mut map_size = shallow_size; + for (k, v) in map.iter() { + map_size += k.size_of(ops); + map_size += v.size_of(ops); + } + + self.labels.size_of(ops) + self.submetric.size_of(ops) + map_size + } +} + /// Sealed traits protect against downstream implementations. /// /// We wrap it in a private module that is inaccessible outside of this module. diff --git a/glean-core/src/metrics/metrics_enabled_config.rs b/glean-core/src/metrics/metrics_enabled_config.rs index b36cbc150a..5c13aae5bf 100644 --- a/glean-core/src/metrics/metrics_enabled_config.rs +++ b/glean-core/src/metrics/metrics_enabled_config.rs @@ -4,6 +4,7 @@ use std::collections::HashMap; +use malloc_size_of_derive::MallocSizeOf; use serde::{Deserialize, Serialize}; /// Represents a list of metrics and an associated boolean property @@ -15,7 +16,7 @@ use serde::{Deserialize, Serialize}; /// "category.metric_name": true /// } /// ``` -#[derive(Serialize, Deserialize, Debug, Clone, Default)] +#[derive(Serialize, Deserialize, Debug, Clone, Default, MallocSizeOf)] pub struct MetricsEnabledConfig { /// This is a `HashMap` consisting of base_identifiers as keys /// and bool values representing an override for the `disabled` diff --git a/glean-core/src/metrics/mod.rs b/glean-core/src/metrics/mod.rs index 92001efd2a..7523f69ecc 100644 --- a/glean-core/src/metrics/mod.rs +++ b/glean-core/src/metrics/mod.rs @@ -8,6 +8,7 @@ use std::collections::HashMap; use std::sync::atomic::Ordering; use chrono::{DateTime, FixedOffset}; +use malloc_size_of::MallocSizeOf; use serde::{Deserialize, Serialize}; use serde_json::json; pub use serde_json::Value as JsonValue; @@ -148,6 +149,32 @@ pub enum Metric { Object(String), } +impl MallocSizeOf for Metric { + fn size_of(&self, ops: &mut malloc_size_of::MallocSizeOfOps) -> usize { + match self { + Metric::Boolean(m) => m.size_of(ops), + Metric::Counter(m) => m.size_of(ops), + // Custom distributions are in the same section, no matter what bucketing. + Metric::CustomDistributionExponential(m) => m.size_of(ops), + Metric::CustomDistributionLinear(m) => m.size_of(ops), + Metric::Datetime(_a, b) => b.size_of(ops), + Metric::Experiment(m) => m.size_of(ops), + Metric::Quantity(m) => m.size_of(ops), + Metric::Rate(a, b) => a.size_of(ops) + b.size_of(ops), + Metric::String(m) => m.size_of(ops), + Metric::StringList(m) => m.size_of(ops), + Metric::Timespan(a, b) => a.size_of(ops) + b.size_of(ops), + Metric::TimingDistribution(m) => m.size_of(ops), + Metric::Url(m) => m.size_of(ops), + Metric::Uuid(m) => m.size_of(ops), + Metric::MemoryDistribution(m) => m.size_of(ops), + Metric::Jwe(m) => m.size_of(ops), + Metric::Text(m) => m.size_of(ops), + Metric::Object(m) => m.size_of(ops), + } + } +} + /// A [`MetricType`] describes common behavior across all metrics. pub trait MetricType { /// Access the stored metadata @@ -292,3 +319,34 @@ impl Metric { } } } + +macro_rules! impl_malloc_size_of_for_metric { + ($ty:ident) => { + impl ::malloc_size_of::MallocSizeOf for $ty { + fn size_of(&self, ops: &mut malloc_size_of::MallocSizeOfOps) -> usize { + // Note: This is behind an `Arc`. + // `size_of` should only be called on the main thread to avoid double-counting. + self.meta().size_of(ops) + } + } + }; +} + +impl_malloc_size_of_for_metric!(BooleanMetric); +impl_malloc_size_of_for_metric!(CounterMetric); +impl_malloc_size_of_for_metric!(CustomDistributionMetric); +impl_malloc_size_of_for_metric!(DatetimeMetric); +impl_malloc_size_of_for_metric!(DenominatorMetric); +impl_malloc_size_of_for_metric!(EventMetric); +impl_malloc_size_of_for_metric!(ExperimentMetric); +impl_malloc_size_of_for_metric!(MemoryDistributionMetric); +impl_malloc_size_of_for_metric!(NumeratorMetric); +impl_malloc_size_of_for_metric!(ObjectMetric); +impl_malloc_size_of_for_metric!(QuantityMetric); +impl_malloc_size_of_for_metric!(RateMetric); +impl_malloc_size_of_for_metric!(StringMetric); +impl_malloc_size_of_for_metric!(StringListMetric); +impl_malloc_size_of_for_metric!(TextMetric); +impl_malloc_size_of_for_metric!(TimespanMetric); +impl_malloc_size_of_for_metric!(UrlMetric); +impl_malloc_size_of_for_metric!(UuidMetric); diff --git a/glean-core/src/metrics/ping.rs b/glean-core/src/metrics/ping.rs index 5defab7a71..50f9a84b4f 100644 --- a/glean-core/src/metrics/ping.rs +++ b/glean-core/src/metrics/ping.rs @@ -9,6 +9,7 @@ use crate::ping::PingMaker; use crate::upload::PingPayload; use crate::Glean; +use malloc_size_of_derive::MallocSizeOf; use uuid::Uuid; /// Stores information about a ping. @@ -18,6 +19,7 @@ use uuid::Uuid; #[derive(Clone)] pub struct PingType(Arc); +#[derive(MallocSizeOf)] struct InnerPing { /// The name of the ping. pub name: String, @@ -51,6 +53,14 @@ impl fmt::Debug for PingType { } } +impl ::malloc_size_of::MallocSizeOf for PingType { + fn size_of(&self, ops: &mut malloc_size_of::MallocSizeOfOps) -> usize { + // Note: This is behind an `Arc`. + // `size_of` should only be called on the main thread to avoid double-counting. + self.0.size_of(ops) + } +} + // IMPORTANT: // // When changing this implementation, make sure all the operations are diff --git a/glean-core/src/metrics/recorded_experiment.rs b/glean-core/src/metrics/recorded_experiment.rs index 8b9dc35d98..a0385e2128 100644 --- a/glean-core/src/metrics/recorded_experiment.rs +++ b/glean-core/src/metrics/recorded_experiment.rs @@ -4,11 +4,12 @@ use std::collections::HashMap; +use malloc_size_of_derive::MallocSizeOf; use serde::{Deserialize, Serialize}; use serde_json::{json, Map as JsonMap, Value as JsonValue}; /// Deserialized experiment data. -#[derive(Clone, Serialize, Deserialize, Debug, PartialEq, Eq)] +#[derive(Clone, Serialize, Deserialize, Debug, PartialEq, Eq, MallocSizeOf)] pub struct RecordedExperiment { /// The experiment's branch as set through [`set_experiment_active`](crate::glean_set_experiment_active). pub branch: String, diff --git a/glean-core/src/metrics/time_unit.rs b/glean-core/src/metrics/time_unit.rs index 6c68d5dff0..e0c44fd911 100644 --- a/glean-core/src/metrics/time_unit.rs +++ b/glean-core/src/metrics/time_unit.rs @@ -4,13 +4,14 @@ use std::time::Duration; +use malloc_size_of_derive::MallocSizeOf; use serde::{Deserialize, Serialize}; use crate::error::{Error, ErrorKind}; /// Different resolutions supported by the time related /// metric types (e.g. DatetimeMetric). -#[derive(Copy, Clone, Debug, Deserialize, Serialize, PartialEq, Eq)] +#[derive(Copy, Clone, Debug, Deserialize, Serialize, PartialEq, Eq, MallocSizeOf)] #[serde(rename_all = "lowercase")] #[repr(i32)] // use i32 to be compatible with our JNA definition pub enum TimeUnit { diff --git a/glean-core/src/metrics/timing_distribution.rs b/glean-core/src/metrics/timing_distribution.rs index 776935afea..af6ada0a62 100644 --- a/glean-core/src/metrics/timing_distribution.rs +++ b/glean-core/src/metrics/timing_distribution.rs @@ -6,6 +6,8 @@ use std::collections::HashMap; use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::{Arc, Mutex}; +use malloc_size_of_derive::MallocSizeOf; + use crate::common_metric_data::CommonMetricDataInternal; use crate::error_recording::{record_error, test_get_num_recorded_errors, ErrorType}; use crate::histogram::{Functional, Histogram}; @@ -34,7 +36,7 @@ const MAX_SAMPLE_TIME: u64 = 1000 * 1000 * 1000 * 60 * 10; /// /// Its internals are considered private, /// but due to UniFFI's behavior we expose its field for now. -#[derive(Copy, Clone, Debug, Eq, PartialEq, Hash)] +#[derive(Copy, Clone, Debug, Eq, PartialEq, Hash, MallocSizeOf)] pub struct TimerId { /// This timer's id. pub id: u64, @@ -63,6 +65,17 @@ pub struct TimingDistributionMetric { start_times: Arc>>, } +impl ::malloc_size_of::MallocSizeOf for TimingDistributionMetric { + fn size_of(&self, ops: &mut malloc_size_of::MallocSizeOfOps) -> usize { + // Note: This is behind an `Arc`. + // `size_of` should only be called on the main thread to avoid double-counting. + self.meta.size_of(ops) + + self.time_unit.size_of(ops) + + self.next_id.size_of(ops) + + self.start_times.lock().unwrap().size_of(ops) + } +} + /// Create a snapshot of the histogram with a time unit. /// /// The snapshot can be serialized into the payload format. diff --git a/glean-core/src/upload/directory.rs b/glean-core/src/upload/directory.rs index 91a4d061d1..97c5c8aeeb 100644 --- a/glean-core/src/upload/directory.rs +++ b/glean-core/src/upload/directory.rs @@ -9,6 +9,8 @@ use std::fs::{self, File}; use std::io::{BufRead, BufReader}; use std::path::{Path, PathBuf}; +use malloc_size_of::MallocSizeOf; +use malloc_size_of_derive::MallocSizeOf; use serde::{Deserialize, Serialize}; use uuid::Uuid; @@ -16,7 +18,7 @@ use super::request::HeaderMap; use crate::{DELETION_REQUEST_PINGS_DIRECTORY, PENDING_PINGS_DIRECTORY}; /// A representation of the data extracted from a ping file, -#[derive(Clone, Debug, Default)] +#[derive(Clone, Debug, Default, MallocSizeOf)] pub struct PingPayload { /// The ping's doc_id. pub document_id: String, @@ -39,6 +41,26 @@ pub struct PingPayloadsByDirectory { pub deletion_request_pings: Vec<(u64, PingPayload)>, } +impl MallocSizeOf for PingPayloadsByDirectory { + fn size_of(&self, ops: &mut malloc_size_of::MallocSizeOfOps) -> usize { + let shallow_size = unsafe { + ops.malloc_size_of(self.pending_pings.as_ptr()) + + ops.malloc_size_of(self.deletion_request_pings.as_ptr()) + }; + + let mut n = shallow_size; + for elem in self.pending_pings.iter() { + n += elem.0.size_of(ops); + n += elem.1.size_of(ops); + } + for elem in self.deletion_request_pings.iter() { + n += elem.0.size_of(ops); + n += elem.1.size_of(ops); + } + n + } +} + impl PingPayloadsByDirectory { /// Extends the data of this instance of PingPayloadsByDirectory /// with the data from another instance of PingPayloadsByDirectory. @@ -104,11 +126,13 @@ fn process_metadata(path: &str, metadata: &str) -> Option { } /// Manages the pings directories. -#[derive(Debug, Clone)] +#[derive(Debug, Clone, MallocSizeOf)] pub struct PingDirectoryManager { /// Path to the pending pings directory. + #[ignore_malloc_size_of = "libstd type"] pending_pings_dir: PathBuf, /// Path to the deletion-request pings directory. + #[ignore_malloc_size_of = "libstd type"] deletion_request_pings_dir: PathBuf, } diff --git a/glean-core/src/upload/mod.rs b/glean-core/src/upload/mod.rs index f217137f00..8538d1e7a8 100644 --- a/glean-core/src/upload/mod.rs +++ b/glean-core/src/upload/mod.rs @@ -14,6 +14,7 @@ use std::collections::HashMap; use std::collections::VecDeque; +use std::mem; use std::path::PathBuf; use std::sync::atomic::{AtomicBool, AtomicU32, Ordering}; use std::sync::{Arc, RwLock, RwLockWriteGuard}; @@ -21,6 +22,8 @@ use std::thread; use std::time::{Duration, Instant}; use chrono::Utc; +use malloc_size_of::MallocSizeOf; +use malloc_size_of_derive::MallocSizeOf; use crate::error::ErrorKind; use crate::TimerId; @@ -40,7 +43,7 @@ mod result; const WAIT_TIME_FOR_PING_PROCESSING: u64 = 1000; // in milliseconds -#[derive(Debug)] +#[derive(Debug, MallocSizeOf)] struct RateLimiter { /// The instant the current interval has started. started: Option, @@ -215,6 +218,44 @@ pub struct PingUploadManager { in_flight: RwLock>, } +impl MallocSizeOf for PingUploadManager { + fn size_of(&self, ops: &mut malloc_size_of::MallocSizeOfOps) -> usize { + let shallow_size = { + let queue = self.queue.read().unwrap(); + if ops.has_malloc_enclosing_size_of() { + if let Some(front) = queue.front() { + // The front element is an interior pointer. + unsafe { ops.malloc_enclosing_size_of(front) } + } else { + // This assumes that no memory is allocated when the VecDeque is empty. + 0 + } + } else { + // An estimate. + queue.capacity() * mem::size_of::() + } + }; + + let mut n = shallow_size + + self.directory_manager.size_of(ops) + // We own this arc + + unsafe { ops.malloc_size_of(self.processed_pending_pings.as_ptr()) } + + self.cached_pings.read().unwrap().size_of(ops) + + self.rate_limiter.as_ref().map(|rl| { + let lock = rl.read().unwrap(); + (*lock).size_of(ops) + }).unwrap_or(0) + + self.language_binding_name.size_of(ops) + + self.upload_metrics.size_of(ops) + + self.policy.size_of(ops); + + let in_flight = self.in_flight.read().unwrap(); + n += in_flight.size_of(ops); + + n + } +} + impl PingUploadManager { /// Creates a new PingUploadManager. /// diff --git a/glean-core/src/upload/policy.rs b/glean-core/src/upload/policy.rs index 91467ebd82..6512648ce9 100644 --- a/glean-core/src/upload/policy.rs +++ b/glean-core/src/upload/policy.rs @@ -4,6 +4,8 @@ //! Policies for ping storage, uploading and requests. +use malloc_size_of_derive::MallocSizeOf; + const MAX_RECOVERABLE_FAILURES: u32 = 3; const MAX_WAIT_ATTEMPTS: u32 = 3; const MAX_PING_BODY_SIZE: usize = 1024 * 1024; // 1 MB @@ -18,7 +20,7 @@ const MAX_PENDING_PINGS_DIRECTORY_SIZE: u64 = 10 * 1024 * 1024; // 10MB const MAX_PENDING_PINGS_COUNT: u64 = 250; /// A struct holding the values for all the policies related to ping storage, uploading and requests. -#[derive(Debug)] +#[derive(Debug, MallocSizeOf)] pub struct Policy { /// The maximum recoverable failures allowed per uploading window. ///