From a374a4391d5b5f2576fa2db6870a9a792ff76a5a Mon Sep 17 00:00:00 2001 From: Mathieu Baudet <1105398+ma2bd@users.noreply.github.com> Date: Wed, 30 Oct 2024 01:09:02 -0400 Subject: [PATCH] Better logging for subscriptions (#2735) * increase logging for subscriptions * Update linera-rpc/src/grpc/client.rs Co-authored-by: Janito Vaqueiro Ferreira Filho Signed-off-by: Mathieu Baudet <1105398+ma2bd@users.noreply.github.com> --- linera-core/src/client/mod.rs | 4 +++- linera-rpc/src/grpc/client.rs | 19 ++++++++++++------- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/linera-core/src/client/mod.rs b/linera-core/src/client/mod.rs index cbd1500fc14..cf8675ed774 100644 --- a/linera-core/src/client/mod.rs +++ b/linera-core/src/client/mod.rs @@ -3210,7 +3210,9 @@ where }) .filter_map(move |result| async move { if let Err(error) = &result { - info!(?error, "Could not connect to validator {name}"); + warn!(?error, "Could not connect to validator {name}"); + } else { + info!("Connected to validator {name}"); } result.ok() }) diff --git a/linera-rpc/src/grpc/client.rs b/linera-rpc/src/grpc/client.rs index e746dfded6a..8c73fd14641 100644 --- a/linera-rpc/src/grpc/client.rs +++ b/linera-rpc/src/grpc/client.rs @@ -67,10 +67,10 @@ impl GrpcClient { /// Returns whether this gRPC status means the server stream should be reconnected to, or not. /// Logs a warning on unexpected status codes. - fn is_retryable(status: &Status) -> bool { + fn is_retryable(status: &Status, address: &str) -> bool { match status.code() { Code::DeadlineExceeded | Code::Aborted | Code::Unavailable | Code::Unknown => { - info!("gRPC request interrupted: {}; retrying", status); + info!("gRPC request to {address} interrupted: {status}; retrying"); true } Code::Ok @@ -78,7 +78,7 @@ impl GrpcClient { | Code::NotFound | Code::AlreadyExists | Code::ResourceExhausted => { - error!("Unexpected gRPC status: {}; retrying", status); + error!("gRPC request to {address} interrupted: {status}; retrying"); true } Code::InvalidArgument @@ -89,7 +89,7 @@ impl GrpcClient { | Code::Internal | Code::DataLoss | Code::Unauthenticated => { - error!("Unexpected gRPC status: {}", status); + error!("Unexpected gRPC status received from {address}: {status}"); false } } @@ -100,6 +100,7 @@ impl GrpcClient { f: F, request: impl TryInto + fmt::Debug + Clone, handler: &str, + address: &str, ) -> Result where F: Fn(ValidatorNodeClient, Request) -> Fut, @@ -113,7 +114,7 @@ impl GrpcClient { })?; loop { match f(self.client.clone(), Request::new(request_inner.clone())).await { - Err(s) if Self::is_retryable(&s) && retry_count < self.max_retries => { + Err(s) if Self::is_retryable(&s, address) && retry_count < self.max_retries => { let delay = self.retry_delay.saturating_mul(retry_count); retry_count += 1; linera_base::time::timer::sleep(delay).await; @@ -121,7 +122,9 @@ impl GrpcClient { } Err(s) => { return Err(NodeError::GrpcError { - error: format!("remote request [{handler}] failed with status: {s:?}",), + error: format!( + "remote request [{handler}] to {address} failed with status: {s:?}", + ), }); } Ok(result) => return Ok(result.into_inner()), @@ -160,6 +163,7 @@ macro_rules! client_delegate { |mut client, req| async move { client.$handler(req).await }, $req, stringify!($handler), + &$self.address, ) .await }}; @@ -257,6 +261,7 @@ impl ValidatorNode for GrpcClient { // The stream of `Notification`s that inserts increasing delays after retriable errors, and // terminates after unexpected or fatal errors. + let address = self.address.clone(); let notification_stream = endlessly_retrying_notification_stream .map(|result| { Option::::try_from(result?).map_err(|err| { @@ -269,7 +274,7 @@ impl ValidatorNode for GrpcClient { retry_count = 0; return future::Either::Left(future::ready(true)); }; - if !Self::is_retryable(status) || retry_count >= max_retries { + if !Self::is_retryable(status, &address) || retry_count >= max_retries { return future::Either::Left(future::ready(false)); } let delay = retry_delay.saturating_mul(retry_count);