Fix issue with NaN measurements in Latency readings

Unfortunately seeing latency measurements returning as NaN (which
panic's when turning them into a Duration). Still working on tracing
down the root cause, but throwing in some checks to mitigate the issue.

balter-core/src/constants.rs

@@ -3,8 +3,5 @@ use std::time::Duration;
 
 pub const BASE_TPS: NonZeroU32 = unsafe { NonZeroU32::new_unchecked(512) };
 pub const BASE_CONCURRENCY: usize = 10;
-pub const BASE_INTERVAL: Duration = Duration::from_millis(500);
-pub const BASE_INTERVAL_SLOW: Duration = Duration::from_millis(4000);
-
-pub const MIN_SAMPLE_COUNT: u64 = 256;
-pub const ADJUSTABLE_SAMPLE_COUNT: u64 = 5_000;
+pub const BASE_INTERVAL: Duration = Duration::from_millis(1000);
+pub const BASE_INTERVAL_SLOW: Duration = Duration::from_millis(5000);

balter/src/measurement.rs

@@ -1,6 +1,7 @@
 use pdatastructs::tdigest::{TDigest, K1};
 use std::fmt;
 use std::time::Duration;
+use tracing::error;
 
 const TDIGEST_BACKLOG_SIZE: usize = 100;
 
@@ -33,6 +34,15 @@ impl Measurement {
 
     pub fn latency(&self, quantile: f64) -> Duration {
         let secs = self.latency.quantile(quantile);
+
+        // TODO: Unfortunately TDigest sometimes returns NaN which we need to filter for.
+        let secs = if secs.is_finite() {
+            secs
+        } else {
+            error!("NaN Latency Calculation. This is a known bug in Balter.");
+            0.
+        };
+
         Duration::from_secs_f64(secs)
     }
 }