From 2e4b33135fc8db6b9caf0a388e99e71cabf39b97 Mon Sep 17 00:00:00 2001
From: Till Rohrmann <till@restate.dev>
Date: Tue, 6 Aug 2024 10:07:39 +0200
Subject: [PATCH] Classify ProposalDropped as Unavailable Tonic status

The raft metadata store does not accept new proposals if there is
no known leader. In this situation, request failed with an internal
ProposalDropped error. This commit changes the behavior so that a
ProposalDropped error will be translated into an unavailable Tonic
status. That way, the request will get automatically retried.
---
 crates/metadata-store/src/grpc/handler.rs |  3 ++-
 crates/metadata-store/src/lib.rs          |  4 +++-
 crates/metadata-store/src/raft/store.rs   | 11 ++++++++++-
 3 files changed, 15 insertions(+), 3 deletions(-)

diff --git a/crates/metadata-store/src/grpc/handler.rs b/crates/metadata-store/src/grpc/handler.rs
index 938e09d8e..54a40224f 100644
--- a/crates/metadata-store/src/grpc/handler.rs
+++ b/crates/metadata-store/src/grpc/handler.rs
@@ -132,7 +132,8 @@ impl MetadataStoreSvc for MetadataStoreHandler {
 impl From<RequestError> for Status {
     fn from(err: RequestError) -> Self {
         match err {
-            RequestError::FailedPrecondition(msg) => Status::failed_precondition(msg.to_string()),
+            RequestError::FailedPrecondition(err) => Status::failed_precondition(err.to_string()),
+            RequestError::Unavailable(err) => Status::unavailable(err.to_string()),
             err => Status::internal(err.to_string()),
         }
     }
diff --git a/crates/metadata-store/src/lib.rs b/crates/metadata-store/src/lib.rs
index 3ab083883..be7c13968 100644
--- a/crates/metadata-store/src/lib.rs
+++ b/crates/metadata-store/src/lib.rs
@@ -32,7 +32,9 @@ pub type RequestReceiver = mpsc::Receiver<MetadataStoreRequest>;
 #[derive(Debug, thiserror::Error)]
 pub enum RequestError {
     #[error("internal error: {0}")]
-    Internal(#[from] GenericError),
+    Internal(GenericError),
+    #[error("service currently unavailable: {0}")]
+    Unavailable(GenericError),
     #[error("failed precondition: {0}")]
     FailedPrecondition(#[from] PreconditionViolation),
     #[error("invalid argument: {0}")]
diff --git a/crates/metadata-store/src/raft/store.rs b/crates/metadata-store/src/raft/store.rs
index 94292a2d3..915597d2e 100644
--- a/crates/metadata-store/src/raft/store.rs
+++ b/crates/metadata-store/src/raft/store.rs
@@ -112,7 +112,7 @@ impl RaftMetadataStore {
                         .map_err(Into::into)
                         .and_then(|request| self.raw_node
                             .propose(vec![], request)
-                            .map_err(|err| RequestError::Internal(err.into()))) {
+                            .map_err(RequestError::from)) {
                         info!("Failed processing request: {err}");
                         callback.fail(err);
                         continue;
@@ -528,3 +528,12 @@ enum RequestKind {
         precondition: Precondition,
     },
 }
+
+impl From<raft::Error> for RequestError {
+    fn from(value: raft::Error) -> Self {
+        match value {
+            err @ raft::Error::ProposalDropped => RequestError::Unavailable(err.into()),
+            err => RequestError::Internal(err.into()),
+        }
+    }
+}