From 4515c833c9379f8c75e756ec677560eb860b51d7 Mon Sep 17 00:00:00 2001
From: Trevor Porter <tkporter4@gmail.com>
Date: Thu, 12 Dec 2024 03:47:13 +0000
Subject: [PATCH] fix: prioritize higher nonces in the message processor
 (#4921)

### Description

Noticed that the iteration logic doesn't actually prioritize higher
nonces. See these example logs
https://cloudlogging.app.goo.gl/CAosszzjFzi3uNJM6 when I sent a message
right after a relayer restart -- it wouldn't move forward on the high
iterator despite there being an unprocessable message on the high
iterator, as it was preferring to iterate the low nonce iter through all
the processed messages (down to nonce 0). This meant it took a really
long time for my message to get relayed!

With this new logic, we should see upon startup we actually do
prioritize new messages, not just high nonce messages from before the
time of restart.

Would also be open to maybe moving forward the low and high iter at the
same time, but that is a bigger change so I'll leave it like this for
now

### Drive-by changes

<!--
Are there any minor or drive-by changes also included?
-->

### Related issues

<!--
- Fixes #[issue number here]
-->

### Backward compatibility

<!--
Are these changes backward compatible? Are there any infrastructure
implications, e.g. changes that would prohibit deploying older commits
using this infra tooling?

Yes/No
-->

### Testing

<!--
What kind of testing have these changes undergone?

None/Manual/Unit Tests
-->
---
 rust/main/agents/relayer/src/msg/processor.rs | 19 ++++++++++++-------
 1 file changed, 12 insertions(+), 7 deletions(-)

diff --git a/rust/main/agents/relayer/src/msg/processor.rs b/rust/main/agents/relayer/src/msg/processor.rs
index 59ec32cb69..21c6b1def1 100644
--- a/rust/main/agents/relayer/src/msg/processor.rs
+++ b/rust/main/agents/relayer/src/msg/processor.rs
@@ -86,24 +86,29 @@ impl ForwardBackwardIterator {
         loop {
             let high_nonce_message_status = self.high_nonce_iter.try_get_next_nonce(metrics)?;
             let low_nonce_message_status = self.low_nonce_iter.try_get_next_nonce(metrics)?;
-            // Always prioritize the high nonce message
+
             match (high_nonce_message_status, low_nonce_message_status) {
-                // Keep iterating if only processed messages are found
+                // Always prioritize advancing the the high nonce iterator, as
+                // we have a preference for higher nonces
                 (MessageStatus::Processed, _) => {
                     self.high_nonce_iter.iterate();
                 }
-                (_, MessageStatus::Processed) => {
-                    self.low_nonce_iter.iterate();
-                }
-                // Otherwise return - either a processable message or nothing to process
                 (MessageStatus::Processable(high_nonce_message), _) => {
                     self.high_nonce_iter.iterate();
                     return Ok(Some(high_nonce_message));
                 }
+
+                // Low nonce messages are only processed if the high nonce iterator
+                // can't make any progress
+                (_, MessageStatus::Processed) => {
+                    self.low_nonce_iter.iterate();
+                }
                 (_, MessageStatus::Processable(low_nonce_message)) => {
                     self.low_nonce_iter.iterate();
                     return Ok(Some(low_nonce_message));
                 }
+
+                // If both iterators give us unindexed messages, there are no messages at the moment
                 (MessageStatus::Unindexed, MessageStatus::Unindexed) => return Ok(None),
             }
             // This loop may iterate through millions of processed messages, blocking the runtime.
@@ -157,7 +162,7 @@ impl DirectionalNonceIterator {
     }
 
     fn try_get_next_nonce(
-        &mut self,
+        &self,
         metrics: &MessageProcessorMetrics,
     ) -> Result<MessageStatus<HyperlaneMessage>> {
         if let Some(message) = self.indexed_message_with_nonce()? {