feat(sdk): detect stale nodes (#2254)

packages/rs-dapi-client/src/dapi_client.rs

@@ -39,11 +39,11 @@ pub enum DapiClientError<TE: Mockable> {
 }
 
 impl<TE: CanRetry + Mockable> CanRetry for DapiClientError<TE> {
-    fn is_node_failure(&self) -> bool {
+    fn can_retry(&self) -> bool {
         use DapiClientError::*;
         match self {
             NoAvailableAddresses => false,
-            Transport(transport_error, _) => transport_error.is_node_failure(),
+            Transport(transport_error, _) => transport_error.can_retry(),
             AddressList(_) => false,
             #[cfg(feature = "mocks")]
             Mock(_) => false,
@@ -233,7 +233,7 @@ impl DapiRequestExecutor for DapiClient {
                         tracing::trace!(?response, "received {} response", response_name);
                     }
                     Err(error) => {
-                        if error.is_node_failure() {
+                        if !error.can_retry() {
                             if applied_settings.ban_failed_address {
                                 let mut address_list = self
                                     .address_list
@@ -264,12 +264,12 @@ impl DapiRequestExecutor for DapiClient {
                     duration.as_secs_f32()
                 )
             })
-            .when(|e| e.is_node_failure())
+            .when(|e| e.can_retry())
             .instrument(tracing::info_span!("request routine"))
             .await;
 
         if let Err(error) = &result {
-            if error.is_node_failure() {
+            if !error.can_retry() {
                 tracing::error!(?error, "request failed");
             }
         }

packages/rs-dapi-client/src/lib.rs

@@ -71,9 +71,16 @@ impl<T: transport::TransportRequest + Send> DapiRequest for T {
     }
 }
 
-/// Allows to flag the transport error variant how tolerant we are of it and whether we can
-/// try to do a request again.
+/// Returns true if the operation can be retried.
 pub trait CanRetry {
+    /// Returns true if the operation can be retried safely.
+    fn can_retry(&self) -> bool;
+
     /// Get boolean flag that indicates if the error is retryable.
-    fn is_node_failure(&self) -> bool;
+    ///
+    /// Depreacted in favor of [CanRetry::can_retry].
+    #[deprecated = "Use !can_retry() instead"]
+    fn is_node_failure(&self) -> bool {
+        !self.can_retry()
+    }
 }

packages/rs-dapi-client/src/transport/grpc.rs

@@ -117,11 +117,12 @@ impl TransportClient for CoreGrpcClient {
 }
 
 impl CanRetry for dapi_grpc::tonic::Status {
-    fn is_node_failure(&self) -> bool {
+    fn can_retry(&self) -> bool {
         let code = self.code();
 
         use dapi_grpc::tonic::Code::*;
-        matches!(
+
+        !matches!(
             code,
             Ok | DataLoss
                 | Cancelled

packages/rs-sdk/Cargo.toml

@@ -6,6 +6,7 @@ edition = "2021"
 [dependencies]
 
 arc-swap = { version = "1.7.1" }
+chrono = { version = "0.4.38" }
 dpp = { path = "../rs-dpp", default-features = false, features = [
   "dash-sdk-features",
 ] }
@@ -52,7 +53,6 @@ data-contracts = { path = "../data-contracts" }
 tokio-test = { version = "0.4.4" }
 clap = { version = "4.5.4", features = ["derive"] }
 sanitize-filename = { version = "0.5.0" }
-chrono = { version = "0.4.38" }
 test-case = { version = "3.3.1" }
 
 [features]

packages/rs-sdk/src/error.rs

@@ -5,7 +5,7 @@ use std::time::Duration;
 use dapi_grpc::mock::Mockable;
 use dpp::version::PlatformVersionError;
 use dpp::ProtocolError;
-use rs_dapi_client::DapiClientError;
+use rs_dapi_client::{CanRetry, DapiClientError};
 
 pub use drive_proof_verifier::error::ContextProviderError;
 
@@ -67,6 +67,10 @@ pub enum Error {
     /// Operation cancelled - cancel token was triggered, timeout, etc.
     #[error("Operation cancelled: {0}")]
     Cancelled(String),
+
+    /// Remote node is stale; try another server
+    #[error(transparent)]
+    StaleNode(#[from] StaleNodeError),
 }
 
 impl<T: Debug + Mockable> From<DapiClientError<T>> for Error {
@@ -80,3 +84,36 @@ impl From<PlatformVersionError> for Error {
         Self::Protocol(value.into())
     }
 }
+
+impl CanRetry for Error {
+    fn can_retry(&self) -> bool {
+        matches!(self, Error::StaleNode(..) | Error::TimeoutReached(_, _))
+    }
+}
+
+/// Server returned stale metadata
+#[derive(Debug, thiserror::Error)]
+pub enum StaleNodeError {
+    /// Server returned metadata with outdated height
+    #[error("received height is outdated: expected {expected_height}, received {received_height}, tolerance {tolerance_blocks}; try another server")]
+    Height {
+        /// Expected height - last block height seen by the Sdk
+        expected_height: u64,
+        /// Block height received from the server
+        received_height: u64,
+        /// Tolerance - how many blocks can be behind the expected height
+        tolerance_blocks: u64,
+    },
+    /// Server returned metadata with time outside of the tolerance
+    #[error(
+        "received invalid time: expected {expected_timestamp_ms}ms, received {received_timestamp_ms} ms, tolerance {tolerance_ms} ms; try another server"
+    )]
+    Time {
+        /// Expected time in milliseconds - is local time when the message was received
+        expected_timestamp_ms: u64,
+        /// Time received from the server in the message, in milliseconds
+        received_timestamp_ms: u64,
+        /// Tolerance in milliseconds
+        tolerance_ms: u64,
+    },
+}