From e3bd8bc5615783e4782da4532b1be7a96bb22289 Mon Sep 17 00:00:00 2001 From: Trevor Porter Date: Fri, 13 Dec 2024 13:28:19 +0000 Subject: [PATCH] feat: ignore unprocessable messages in metrics, add some new warp monitoring (#4995) ### Description - Ignores known unprocessable ezETH and pzETH messages from metrics, to avoid alerts there - Added some logic to remove stale warp route monitors if a warp ID changes such that the previous warp monitor is stale - Deployed new warp monitors for ezETH and pzETH to include swell - Pulled in the treasure warp route ID that wasn't put into a PR yet ### Drive-by changes ### Related issues ### Backward compatibility ### Testing --------- Co-authored-by: Mo Hussan <22501692+Mo-Hussain@users.noreply.github.com> --- .../config/environments/mainnet3/agent.ts | 41 +++++++++++- .../warp-routes/deploy-warp-monitor.ts | 6 ++ typescript/infra/src/utils/helm.ts | 5 ++ typescript/infra/src/warp/helm.ts | 62 +++++++++++++++++-- typescript/sdk/src/metadata/matchingList.ts | 1 + 5 files changed, 109 insertions(+), 6 deletions(-) diff --git a/typescript/infra/config/environments/mainnet3/agent.ts b/typescript/infra/config/environments/mainnet3/agent.ts index ad075baccc..362adb4df8 100644 --- a/typescript/infra/config/environments/mainnet3/agent.ts +++ b/typescript/infra/config/environments/mainnet3/agent.ts @@ -1,6 +1,7 @@ import { GasPaymentEnforcement, GasPaymentEnforcementPolicyType, + MatchingList, RpcConsensusType, } from '@hyperlane-xyz/sdk'; @@ -458,6 +459,41 @@ const scraperResources = { }, }; +const blacklistedMessageIds = [ + // ezETH + '0xb9cfeb4a22b65903ca7cb514fd752feba0622a0495878d508d19a91734d89cc4', + '0x13d6c56781ee9b8811f4e17198bf064baed2682ce44193c750e76c73384466e7', + '0x366520dcd48f19a2cdc806e244d4cea970a587e3932320baee30e710d316b303', + '0x0f9b8849d6dbf5a699e906a6e06044d6cf84ee0ba2174cec28db4fceba52616a', + '0x0e1235105208e7d3a616ac2bb780e7dab30fc289670ba8d6655a4ded73f9b5da', + '0xa6fdecc3f21d081bf3d78da9ddf516b24397a6bff44d7cd4614955f5ca2320b2', + '0x2c3484724a97524fd95aa8aec34a0ae30f79e14e1b228cce9dc1793cea40fc3d', + '0x11ffaeaae5c431501584bc39805ef44b4080e7f90ca7ff609a131d58d1f75ae6', + '0xc18ea74675bc1e5b780e63ac6063c7c39189e1848b8fe52ac40b83fff9268483', + '0xd8040094ab94e44e2b3b57ab0704a33e363f46261a45c9dfc788371c808b8f3a', + '0xf7f0be22f46144793ee3fadccddd4cfb8422d36f5d59bb86fea3782b89160d49', + '0xeda79ab37b4a05d8f318b3a465a70572d819b2c37456c48835a30bb6c016e194', + '0xaf7c7dfc4d19aec283c619a2724d03fbbfeef4a468e84c0573551c1adca40ded', + '0x4a2c42c283755400c0dc7f1be65f6ff026a38aacaa6505302d465268bcd86b21', + '0x0f80e5b8da5a706d6273a622a5c29f83cee5f37e6376c2c8a615b0ef91a540df', + '0x6359232ef1f239d9519104cf47f1e2fbcbe25f8ee68001c5eff7e81bf23b396c', + '0x6a3fb736b952467b814e93fb35edf3a824d35efd1e4b10e3ed465595c55af88a', + + // pzETH + '0x14cb552c08de9f131b750c2f821f90e5ff685e1d3d714e912f7603b2f4b7adb4', + '0xaa5b5021200e66b4a47e5156106c46b6b2bc1e00b088a524a14bb0709cbf733e', + '0x43b4cf52255a7728a3c409f76fd20ba0c36cb42854e0b0a0eefdde848363224b', + '0x047f34405014b117dccd6d8981c846dc3fe746f5e758f90f227581c735f4f11a', + '0x47d60c21abefae928d1c16c5a33cd5a8fcf870cf533c71ab6db49d75a5c4a215', + '0xa2df671fbd4b518c282f9a21e2677fa2a05af33f96ccc9ff113f1a1ffa557667', + '0x1cefa98b6d937333e452a0dbc0654e13416c228682837a8913cb18d612b307dd', +]; + +// Blacklist matching list intended to be used by all contexts. +const blacklist: MatchingList = blacklistedMessageIds.map((messageId) => ({ + messageId, +})); + const hyperlane: RootAgentConfig = { ...contextBase, context: Contexts.Hyperlane, @@ -467,8 +503,9 @@ const hyperlane: RootAgentConfig = { rpcConsensusType: RpcConsensusType.Fallback, docker: { repo, - tag: 'a7f3967-20241205-163733', + tag: 'c351026-20241212-172408', }, + blacklist, gasPaymentEnforcement: gasPaymentEnforcement, metricAppContextsGetter, resources: relayerResources, @@ -503,6 +540,7 @@ const releaseCandidate: RootAgentConfig = { repo, tag: '4cb2c9a-20241205-142854', }, + blacklist, // We're temporarily (ab)using the RC relayer as a way to increase // message throughput. // whitelist: releaseCandidateHelloworldMatchingList, @@ -536,6 +574,7 @@ const neutron: RootAgentConfig = { repo, tag: '25a927d-20241114-171323', }, + blacklist, gasPaymentEnforcement, metricAppContextsGetter, resources: relayerResources, diff --git a/typescript/infra/scripts/warp-routes/deploy-warp-monitor.ts b/typescript/infra/scripts/warp-routes/deploy-warp-monitor.ts index 29fd16b9a9..8ede7cdda5 100644 --- a/typescript/infra/scripts/warp-routes/deploy-warp-monitor.ts +++ b/typescript/infra/scripts/warp-routes/deploy-warp-monitor.ts @@ -35,6 +35,12 @@ async function main() { await helmManager.runHelmCommand(HelmCommand.InstallOrUpgrade); }; + // First try to uninstall any stale Warp Monitors. + // This can happen if a Warp Route ID is changed or removed. + await WarpRouteMonitorHelmManager.uninstallUnknownWarpMonitorReleases( + environment, + ); + for (const id of warpRouteIds) { console.log(`Deploying Warp Monitor for Warp Route ID: ${id}`); await deployWarpMonitor(id); diff --git a/typescript/infra/src/utils/helm.ts b/typescript/infra/src/utils/helm.ts index 2ec09385ea..855393473a 100644 --- a/typescript/infra/src/utils/helm.ts +++ b/typescript/infra/src/utils/helm.ts @@ -93,6 +93,11 @@ export function buildHelmChartDependencies(chartPath: string) { return execCmd(`cd ${chartPath} && helm dependency build`, {}, false, true); } +// Convenience function to remove a helm release without having a HelmManger for it. +export function removeHelmRelease(releaseName: string, namespace: string) { + return execCmd(`helm uninstall ${releaseName} --namespace ${namespace}`); +} + export type HelmValues = Record; export abstract class HelmManager { diff --git a/typescript/infra/src/warp/helm.ts b/typescript/infra/src/warp/helm.ts index ce6c901203..c77b3e45fe 100644 --- a/typescript/infra/src/warp/helm.ts +++ b/typescript/infra/src/warp/helm.ts @@ -1,10 +1,16 @@ +import { confirm } from '@inquirer/prompts'; import path from 'path'; +import { difference } from '@hyperlane-xyz/utils'; + +import { WarpRouteIds } from '../../config/environments/mainnet3/warp/warpIds.js'; import { DeployEnvironment } from '../../src/config/environment.js'; -import { HelmManager } from '../../src/utils/helm.js'; -import { getInfraPath } from '../../src/utils/utils.js'; +import { HelmManager, removeHelmRelease } from '../../src/utils/helm.js'; +import { execCmdAndParseJson, getInfraPath } from '../../src/utils/utils.js'; export class WarpRouteMonitorHelmManager extends HelmManager { + static helmReleasePrefix: string = 'hyperlane-warp-route-'; + readonly helmChartPath: string = path.join( getInfraPath(), './helm/warp-routes', @@ -22,7 +28,7 @@ export class WarpRouteMonitorHelmManager extends HelmManager { return { image: { repository: 'gcr.io/abacus-labs-dev/hyperlane-monorepo', - tag: 'aac6787-20241128-103715', + tag: 'fd20bb1-20241212-220536', }, warpRouteId: this.warpRouteId, fullnameOverride: this.helmReleaseName, @@ -37,8 +43,12 @@ export class WarpRouteMonitorHelmManager extends HelmManager { return this.runEnv; } - get helmReleaseName(): string { - let name = `hyperlane-warp-route-${this.warpRouteId + get helmReleaseName() { + return WarpRouteMonitorHelmManager.getHelmReleaseName(this.warpRouteId); + } + + static getHelmReleaseName(warpRouteId: string): string { + let name = `${WarpRouteMonitorHelmManager.helmReleasePrefix}${warpRouteId .toLowerCase() .replaceAll('/', '-')}`; @@ -53,4 +63,46 @@ export class WarpRouteMonitorHelmManager extends HelmManager { } return name; } + + // Gets all Warp Monitor Helm Releases in the given namespace. + static async getWarpMonitorHelmReleases( + namespace: string, + ): Promise { + const results = await execCmdAndParseJson( + `helm list --filter '${WarpRouteMonitorHelmManager.helmReleasePrefix}.+' -o json -n ${namespace}`, + ); + return results.map((r: any) => r.name); + } + + // This method is used to uninstall any stale Warp Monitors. + // This can happen if a Warp Route ID is changed or removed. + // Any warp monitor helm releases found that do not relate to known warp route ids + // will be prompted for uninstallation. + static async uninstallUnknownWarpMonitorReleases(namespace: string) { + const allExpectedHelmReleaseNames = Object.values(WarpRouteIds).map( + WarpRouteMonitorHelmManager.getHelmReleaseName, + ); + const helmReleases = + await WarpRouteMonitorHelmManager.getWarpMonitorHelmReleases(namespace); + + const unknownHelmReleases = difference( + new Set(helmReleases), + new Set(allExpectedHelmReleaseNames), + ); + for (const helmRelease of unknownHelmReleases) { + console.log( + `Unknown Warp Monitor Helm Release: ${helmRelease} (possibly a release from a stale Warp Route ID).`, + ); + const uninstall = await confirm({ + message: + "Would you like to uninstall this Helm Release? Make extra sure it shouldn't exist!", + }); + if (uninstall) { + console.log(`Uninstalling Helm Release: ${helmRelease}`); + await removeHelmRelease(helmRelease, namespace); + } else { + console.log(`Skipping uninstall of Helm Release: ${helmRelease}`); + } + } + } } diff --git a/typescript/sdk/src/metadata/matchingList.ts b/typescript/sdk/src/metadata/matchingList.ts index 0e7a50804d..297bc54613 100644 --- a/typescript/sdk/src/metadata/matchingList.ts +++ b/typescript/sdk/src/metadata/matchingList.ts @@ -15,6 +15,7 @@ const DomainSchema = z.union([ const AddressSchema = z.union([z.literal('*'), ZHash, z.array(ZHash)]); const MatchingListElementSchema = z.object({ + messageId: AddressSchema.optional(), originDomain: DomainSchema.optional(), senderAddress: AddressSchema.optional(), destinationDomain: DomainSchema.optional(),