From 22fa8ebd85daedeaa492f1c63b5d0e0896fc6643 Mon Sep 17 00:00:00 2001 From: Byron Wasti Date: Mon, 1 Jul 2024 16:51:43 -0400 Subject: [PATCH] Fix issue with NaN measurements in Latency readings Unfortunately seeing latency measurements returning as NaN (which panic's when turning them into a Duration). Still working on tracing down the root cause, but throwing in some checks to mitigate the issue. --- balter-core/src/constants.rs | 7 ++----- balter/src/measurement.rs | 10 ++++++++++ 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/balter-core/src/constants.rs b/balter-core/src/constants.rs index cad108a..bda02d5 100644 --- a/balter-core/src/constants.rs +++ b/balter-core/src/constants.rs @@ -3,8 +3,5 @@ use std::time::Duration; pub const BASE_TPS: NonZeroU32 = unsafe { NonZeroU32::new_unchecked(512) }; pub const BASE_CONCURRENCY: usize = 10; -pub const BASE_INTERVAL: Duration = Duration::from_millis(500); -pub const BASE_INTERVAL_SLOW: Duration = Duration::from_millis(4000); - -pub const MIN_SAMPLE_COUNT: u64 = 256; -pub const ADJUSTABLE_SAMPLE_COUNT: u64 = 5_000; +pub const BASE_INTERVAL: Duration = Duration::from_millis(1000); +pub const BASE_INTERVAL_SLOW: Duration = Duration::from_millis(5000); diff --git a/balter/src/measurement.rs b/balter/src/measurement.rs index 2dc91ae..6c03364 100644 --- a/balter/src/measurement.rs +++ b/balter/src/measurement.rs @@ -1,6 +1,7 @@ use pdatastructs::tdigest::{TDigest, K1}; use std::fmt; use std::time::Duration; +use tracing::error; const TDIGEST_BACKLOG_SIZE: usize = 100; @@ -33,6 +34,15 @@ impl Measurement { pub fn latency(&self, quantile: f64) -> Duration { let secs = self.latency.quantile(quantile); + + // TODO: Unfortunately TDigest sometimes returns NaN which we need to filter for. + let secs = if secs.is_finite() { + secs + } else { + error!("NaN Latency Calculation. This is a known bug in Balter."); + 0. + }; + Duration::from_secs_f64(secs) } }