Skip to content

Commit

Permalink
feat: add optional instrumentation to readers (#1431)
Browse files Browse the repository at this point in the history
Add optional instrumentation using the `tracing` crate for `vortex-io`
and `vortex-file`. Disabled by default.

Part of some work to make it easier to collect deep profiles of
performance on object storage. Example output when hooked up to a
subscriber:


<img width="1265" alt="image"
src="https://github.com/user-attachments/assets/352564c6-9c99-408b-bb63-4e6c5bed8efd">
  • Loading branch information
a10y authored Nov 21, 2024
1 parent e7cbad6 commit 0d4e8a8
Show file tree
Hide file tree
Showing 7 changed files with 11 additions and 2 deletions.
2 changes: 2 additions & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -135,6 +135,7 @@ tar = "0.4"
tempfile = "3"
thiserror = "2.0.0"
tokio = "1.37.0"
tracing = "0.1"
uninit = "0.6.2"
url = "2"
uuid = "1.8.0"
Expand Down
2 changes: 2 additions & 0 deletions vortex-file/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ tokio = { workspace = true, features = [
"fs",
"rt-multi-thread",
] }
tracing = { workspace = true, optional = true }
vortex-array = { workspace = true }
vortex-buffer = { workspace = true }
vortex-dtype = { workspace = true, features = ["flatbuffers"] }
Expand Down Expand Up @@ -57,3 +58,4 @@ object_store = [
"vortex-error/object_store",
"vortex-io/object_store",
]
tracing = ["dep:tracing", "vortex-io/tracing"]
1 change: 1 addition & 0 deletions vortex-file/src/read/stream.rs
Original file line number Diff line number Diff line change
Expand Up @@ -298,6 +298,7 @@ impl<R: VortexReadAt + Unpin> VortexFileArrayStream<R> {
}
}

#[cfg_attr(feature = "tracing", tracing::instrument(skip_all))]
async fn read_ranges<R: VortexReadAt>(
reader: R,
ranges: Vec<MessageLocator>,
Expand Down
3 changes: 1 addition & 2 deletions vortex-io/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ categories.workspace = true
bytes = { workspace = true }
compio = { workspace = true, features = ["bytes", "macros"], optional = true }
tokio = { workspace = true, features = ["fs"], optional = true }
tracing = { workspace = true, optional = true }
futures = { workspace = true, features = ["std"] }
futures-util = { workspace = true }
flume = { workspace = true }
Expand All @@ -31,8 +32,6 @@ tokio = { workspace = true, features = ["full"] }

[features]
futures = ["futures-util/io"]
compio = ["dep:compio"]
tokio = ["dep:tokio"]
object_store = ["dep:object_store", "vortex-error/object_store"]

[lints]
Expand Down
2 changes: 2 additions & 0 deletions vortex-io/src/object_store.rs
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,7 @@ impl ObjectStoreReadAt {
}

impl VortexReadAt for ObjectStoreReadAt {
#[cfg_attr(feature = "tracing", tracing::instrument(skip(self)))]
fn read_byte_range(
&self,
pos: u64,
Expand All @@ -82,6 +83,7 @@ impl VortexReadAt for ObjectStoreReadAt {
})
}

#[cfg_attr(feature = "tracing", tracing::instrument(skip(self)))]
fn size(&self) -> impl Future<Output = u64> + 'static {
let object_store = self.object_store.clone();
let location = self.location.clone();
Expand Down
2 changes: 2 additions & 0 deletions vortex-io/src/tokio.rs
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,7 @@ impl Deref for TokioFile {
}

impl VortexReadAt for TokioFile {
#[cfg_attr(feature = "tracing", tracing::instrument(skip(self)))]
fn read_byte_range(
&self,
pos: u64,
Expand All @@ -79,6 +80,7 @@ impl VortexReadAt for TokioFile {
}
}

#[cfg_attr(feature = "tracing", tracing::instrument(skip(self)))]
fn size(&self) -> impl Future<Output = u64> + 'static {
let this = self.clone();

Expand Down

0 comments on commit 0d4e8a8

Please sign in to comment.