From ea4035622ec36a1c5f7222412b094a8cfb211b2b Mon Sep 17 00:00:00 2001 From: Daniel Savu <23065004+daniel-savu@users.noreply.github.com> Date: Wed, 15 Nov 2023 17:48:33 +0300 Subject: [PATCH 1/2] feat: cw run-locally termination invariants --- rust/chains/hyperlane-cosmos/src/mailbox.rs | 2 +- rust/utils/run-locally/src/cosmos/mod.rs | 74 ++++++++++++++++++++- 2 files changed, 72 insertions(+), 4 deletions(-) diff --git a/rust/chains/hyperlane-cosmos/src/mailbox.rs b/rust/chains/hyperlane-cosmos/src/mailbox.rs index f4098417a9..6fd36cc655 100644 --- a/rust/chains/hyperlane-cosmos/src/mailbox.rs +++ b/rust/chains/hyperlane-cosmos/src/mailbox.rs @@ -253,7 +253,7 @@ impl CosmosMailboxIndexer { /// The message dispatch event type from the CW contract. const MESSAGE_DISPATCH_EVENT_TYPE: &str = "mailbox_dispatch"; - /// Create a reference to a mailbox at a specific Ethereum address on some + /// Create a reference to a mailbox at a specific Cosmos address on some /// chain pub fn new( conf: ConnectionConf, diff --git a/rust/utils/run-locally/src/cosmos/mod.rs b/rust/utils/run-locally/src/cosmos/mod.rs index 3bdf82f20b..e443bee0d4 100644 --- a/rust/utils/run-locally/src/cosmos/mod.rs +++ b/rust/utils/run-locally/src/cosmos/mod.rs @@ -1,12 +1,13 @@ use std::collections::BTreeMap; use std::path::{Path, PathBuf}; use std::thread::sleep; -use std::time::Duration; +use std::time::{Duration, Instant}; use std::{env, fs}; use cosmwasm_schema::cw_serde; use hpl_interface::types::bech32_decode; use macro_rules_attribute::apply; +use maplit::hashmap; use tempfile::tempdir; mod cli; @@ -26,7 +27,7 @@ use crate::cosmos::link::link_networks; use crate::logging::log; use crate::program::Program; use crate::utils::{as_task, concat_path, stop_child, AgentHandles, TaskHandle}; -use crate::AGENT_BIN_PATH; +use crate::{fetch_metric, AGENT_BIN_PATH}; use cli::{OsmosisCLI, OsmosisEndpoint}; use self::deploy::deploy_cw_hyperlane; @@ -290,6 +291,7 @@ fn launch_cosmos_relayer( .hyp_env("ALLOWLOCALCHECKPOINTSYNCERS", "true") .hyp_env("TRACING_LEVEL", if debug { "debug" } else { "info" }) .hyp_env("GASPAYMENTENFORCEMENT", "[{\"type\": \"none\"}]") + .hyp_env("METRICSPORT", 9093.to_string()) .spawn("RLY"); relayer @@ -300,6 +302,7 @@ const ENV_CW_HYPERLANE_PATH_KEY: &str = "E2E_CW_HYPERLANE_PATH"; #[allow(dead_code)] fn run_locally() { + const TIMEOUT_SECS: u64 = 60 * 10; let debug = false; log!("Building rust..."); @@ -504,7 +507,72 @@ fn run_locally() { relayer: hpl_rly.join(), }; - sleep(Duration::from_secs(100)); // wait for a long time + // Mostly copy-pasta from `rust/utils/run-locally/src/main.rs` + // TODO: refactor to share code + let loop_start = Instant::now(); + // give things a chance to fully start. + sleep(Duration::from_secs(5)); + let mut failure_occurred = false; + loop { + // look for the end condition. + if termination_invariants_met().unwrap_or(false) { + // end condition reached successfully + break; + } else if (Instant::now() - loop_start).as_secs() > TIMEOUT_SECS { + // we ran out of time + log!("timeout reached before message submission was confirmed"); + failure_occurred = true; + break; + } + + sleep(Duration::from_secs(5)); + } + + if failure_occurred { + panic!("E2E tests failed"); + } else { + log!("E2E tests passed"); + } +} + +fn termination_invariants_met() -> eyre::Result { + const COSMOS_MESSAGES_EXPECTED: u32 = 2; + + let gas_payments_scraped = fetch_metric( + "9093", + "hyperlane_contract_sync_stored_events", + &hashmap! {"data_type" => "gas_payment"}, + )? + .iter() + .sum::(); + let expected_gas_payments = COSMOS_MESSAGES_EXPECTED; + if gas_payments_scraped != expected_gas_payments { + log!( + "Scraper has scraped {} gas payments, expected {}", + gas_payments_scraped, + expected_gas_payments + ); + return Ok(false); + } + + let delivered_messages_scraped = fetch_metric( + "9093", + "hyperlane_operations_processed_count", + &hashmap! {"phase" => "confirmed"}, + )? + .iter() + .sum::(); + if delivered_messages_scraped != COSMOS_MESSAGES_EXPECTED { + log!( + "Relayer confirmed {} submitted messages, expected {}", + delivered_messages_scraped, + COSMOS_MESSAGES_EXPECTED + ); + return Ok(false); + } + + log!("Termination invariants have been meet"); + Ok(true) } #[cfg(test)] From fe33d7b51910e80377324f09c47d7e5a9f22348a Mon Sep 17 00:00:00 2001 From: Daniel Savu <23065004+daniel-savu@users.noreply.github.com> Date: Wed, 15 Nov 2023 18:00:30 +0300 Subject: [PATCH 2/2] chore: do not hardcode expected messages --- rust/utils/run-locally/src/cosmos/mod.rs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/rust/utils/run-locally/src/cosmos/mod.rs b/rust/utils/run-locally/src/cosmos/mod.rs index e443bee0d4..01e3864edc 100644 --- a/rust/utils/run-locally/src/cosmos/mod.rs +++ b/rust/utils/run-locally/src/cosmos/mod.rs @@ -456,6 +456,7 @@ fn run_locally() { ); // dispatch messages + let mut dispatched_messages = 0; for node in nodes.iter() { let targets = nodes @@ -472,6 +473,7 @@ fn run_locally() { } for target in targets { + dispatched_messages += 1; let cli = OsmosisCLI::new( osmosisd.clone(), node.launch_resp.home_path.to_str().unwrap(), @@ -515,7 +517,7 @@ fn run_locally() { let mut failure_occurred = false; loop { // look for the end condition. - if termination_invariants_met().unwrap_or(false) { + if termination_invariants_met(dispatched_messages).unwrap_or(false) { // end condition reached successfully break; } else if (Instant::now() - loop_start).as_secs() > TIMEOUT_SECS { @@ -535,9 +537,7 @@ fn run_locally() { } } -fn termination_invariants_met() -> eyre::Result { - const COSMOS_MESSAGES_EXPECTED: u32 = 2; - +fn termination_invariants_met(messages_expected: u32) -> eyre::Result { let gas_payments_scraped = fetch_metric( "9093", "hyperlane_contract_sync_stored_events", @@ -545,7 +545,7 @@ fn termination_invariants_met() -> eyre::Result { )? .iter() .sum::(); - let expected_gas_payments = COSMOS_MESSAGES_EXPECTED; + let expected_gas_payments = messages_expected; if gas_payments_scraped != expected_gas_payments { log!( "Scraper has scraped {} gas payments, expected {}", @@ -562,11 +562,11 @@ fn termination_invariants_met() -> eyre::Result { )? .iter() .sum::(); - if delivered_messages_scraped != COSMOS_MESSAGES_EXPECTED { + if delivered_messages_scraped != messages_expected { log!( "Relayer confirmed {} submitted messages, expected {}", delivered_messages_scraped, - COSMOS_MESSAGES_EXPECTED + messages_expected ); return Ok(false); }