feat: ignore unprocessable messages in metrics, add some new warp mon…

…itoring (#4995)

### Description

- Ignores known unprocessable ezETH and pzETH messages from metrics, to
avoid alerts there
- Added some logic to remove stale warp route monitors if a warp ID
changes such that the previous warp monitor is stale
- Deployed new warp monitors for ezETH and pzETH to include swell
- Pulled in the treasure warp route ID that wasn't put into a PR yet

### Drive-by changes

<!--
Are there any minor or drive-by changes also included?
-->

### Related issues

<!--
- Fixes #[issue number here]
-->

### Backward compatibility

<!--
Are these changes backward compatible? Are there any infrastructure
implications, e.g. changes that would prohibit deploying older commits
using this infra tooling?

Yes/No
-->

### Testing

<!--
What kind of testing have these changes undergone?

None/Manual/Unit Tests
-->

---------

Co-authored-by: Mo Hussan <[email protected]>

typescript/infra/config/environments/mainnet3/agent.ts

@@ -1,6 +1,7 @@
 import {
   GasPaymentEnforcement,
   GasPaymentEnforcementPolicyType,
+  MatchingList,
   RpcConsensusType,
 } from '@hyperlane-xyz/sdk';
 
@@ -458,6 +459,41 @@ const scraperResources = {
   },
 };
 
+const blacklistedMessageIds = [
+  // ezETH
+  '0xb9cfeb4a22b65903ca7cb514fd752feba0622a0495878d508d19a91734d89cc4',
+  '0x13d6c56781ee9b8811f4e17198bf064baed2682ce44193c750e76c73384466e7',
+  '0x366520dcd48f19a2cdc806e244d4cea970a587e3932320baee30e710d316b303',
+  '0x0f9b8849d6dbf5a699e906a6e06044d6cf84ee0ba2174cec28db4fceba52616a',
+  '0x0e1235105208e7d3a616ac2bb780e7dab30fc289670ba8d6655a4ded73f9b5da',
+  '0xa6fdecc3f21d081bf3d78da9ddf516b24397a6bff44d7cd4614955f5ca2320b2',
+  '0x2c3484724a97524fd95aa8aec34a0ae30f79e14e1b228cce9dc1793cea40fc3d',
+  '0x11ffaeaae5c431501584bc39805ef44b4080e7f90ca7ff609a131d58d1f75ae6',
+  '0xc18ea74675bc1e5b780e63ac6063c7c39189e1848b8fe52ac40b83fff9268483',
+  '0xd8040094ab94e44e2b3b57ab0704a33e363f46261a45c9dfc788371c808b8f3a',
+  '0xf7f0be22f46144793ee3fadccddd4cfb8422d36f5d59bb86fea3782b89160d49',
+  '0xeda79ab37b4a05d8f318b3a465a70572d819b2c37456c48835a30bb6c016e194',
+  '0xaf7c7dfc4d19aec283c619a2724d03fbbfeef4a468e84c0573551c1adca40ded',
+  '0x4a2c42c283755400c0dc7f1be65f6ff026a38aacaa6505302d465268bcd86b21',
+  '0x0f80e5b8da5a706d6273a622a5c29f83cee5f37e6376c2c8a615b0ef91a540df',
+  '0x6359232ef1f239d9519104cf47f1e2fbcbe25f8ee68001c5eff7e81bf23b396c',
+  '0x6a3fb736b952467b814e93fb35edf3a824d35efd1e4b10e3ed465595c55af88a',
+
+  // pzETH
+  '0x14cb552c08de9f131b750c2f821f90e5ff685e1d3d714e912f7603b2f4b7adb4',
+  '0xaa5b5021200e66b4a47e5156106c46b6b2bc1e00b088a524a14bb0709cbf733e',
+  '0x43b4cf52255a7728a3c409f76fd20ba0c36cb42854e0b0a0eefdde848363224b',
+  '0x047f34405014b117dccd6d8981c846dc3fe746f5e758f90f227581c735f4f11a',
+  '0x47d60c21abefae928d1c16c5a33cd5a8fcf870cf533c71ab6db49d75a5c4a215',
+  '0xa2df671fbd4b518c282f9a21e2677fa2a05af33f96ccc9ff113f1a1ffa557667',
+  '0x1cefa98b6d937333e452a0dbc0654e13416c228682837a8913cb18d612b307dd',
+];
+
+// Blacklist matching list intended to be used by all contexts.
+const blacklist: MatchingList = blacklistedMessageIds.map((messageId) => ({
+  messageId,
+}));
+
 const hyperlane: RootAgentConfig = {
   ...contextBase,
   context: Contexts.Hyperlane,
@@ -467,8 +503,9 @@ const hyperlane: RootAgentConfig = {
     rpcConsensusType: RpcConsensusType.Fallback,
     docker: {
       repo,
-      tag: 'a7f3967-20241205-163733',
+      tag: 'c351026-20241212-172408',
     },
+    blacklist,
     gasPaymentEnforcement: gasPaymentEnforcement,
     metricAppContextsGetter,
     resources: relayerResources,
@@ -503,6 +540,7 @@ const releaseCandidate: RootAgentConfig = {
       repo,
       tag: '4cb2c9a-20241205-142854',
     },
+    blacklist,
     // We're temporarily (ab)using the RC relayer as a way to increase
     // message throughput.
     // whitelist: releaseCandidateHelloworldMatchingList,
@@ -536,6 +574,7 @@ const neutron: RootAgentConfig = {
       repo,
       tag: '25a927d-20241114-171323',
     },
+    blacklist,
     gasPaymentEnforcement,
     metricAppContextsGetter,
     resources: relayerResources,

typescript/infra/scripts/warp-routes/deploy-warp-monitor.ts

@@ -35,6 +35,12 @@ async function main() {
     await helmManager.runHelmCommand(HelmCommand.InstallOrUpgrade);
   };
 
+  // First try to uninstall any stale Warp Monitors.
+  // This can happen if a Warp Route ID is changed or removed.
+  await WarpRouteMonitorHelmManager.uninstallUnknownWarpMonitorReleases(
+    environment,
+  );
+
   for (const id of warpRouteIds) {
     console.log(`Deploying Warp Monitor for Warp Route ID: ${id}`);
     await deployWarpMonitor(id);

typescript/infra/src/utils/helm.ts

@@ -93,6 +93,11 @@ export function buildHelmChartDependencies(chartPath: string) {
   return execCmd(`cd ${chartPath} && helm dependency build`, {}, false, true);
 }
 
+// Convenience function to remove a helm release without having a HelmManger for it.
+export function removeHelmRelease(releaseName: string, namespace: string) {
+  return execCmd(`helm uninstall ${releaseName} --namespace ${namespace}`);
+}
+
 export type HelmValues = Record<string, any>;
 
 export abstract class HelmManager<T = HelmValues> {

typescript/infra/src/warp/helm.ts

@@ -1,10 +1,16 @@
+import { confirm } from '@inquirer/prompts';
 import path from 'path';
 
+import { difference } from '@hyperlane-xyz/utils';
+
+import { WarpRouteIds } from '../../config/environments/mainnet3/warp/warpIds.js';
 import { DeployEnvironment } from '../../src/config/environment.js';
-import { HelmManager } from '../../src/utils/helm.js';
-import { getInfraPath } from '../../src/utils/utils.js';
+import { HelmManager, removeHelmRelease } from '../../src/utils/helm.js';
+import { execCmdAndParseJson, getInfraPath } from '../../src/utils/utils.js';
 
 export class WarpRouteMonitorHelmManager extends HelmManager {
+  static helmReleasePrefix: string = 'hyperlane-warp-route-';
+
   readonly helmChartPath: string = path.join(
     getInfraPath(),
     './helm/warp-routes',
@@ -22,7 +28,7 @@ export class WarpRouteMonitorHelmManager extends HelmManager {
     return {
       image: {
         repository: 'gcr.io/abacus-labs-dev/hyperlane-monorepo',
-        tag: 'aac6787-20241128-103715',
+        tag: 'fd20bb1-20241212-220536',
       },
       warpRouteId: this.warpRouteId,
       fullnameOverride: this.helmReleaseName,
@@ -37,8 +43,12 @@ export class WarpRouteMonitorHelmManager extends HelmManager {
     return this.runEnv;
   }
 
-  get helmReleaseName(): string {
-    let name = `hyperlane-warp-route-${this.warpRouteId
+  get helmReleaseName() {
+    return WarpRouteMonitorHelmManager.getHelmReleaseName(this.warpRouteId);
+  }
+
+  static getHelmReleaseName(warpRouteId: string): string {
+    let name = `${WarpRouteMonitorHelmManager.helmReleasePrefix}${warpRouteId
       .toLowerCase()
       .replaceAll('/', '-')}`;
 
@@ -53,4 +63,46 @@ export class WarpRouteMonitorHelmManager extends HelmManager {
     }
     return name;
   }
+
+  // Gets all Warp Monitor Helm Releases in the given namespace.
+  static async getWarpMonitorHelmReleases(
+    namespace: string,
+  ): Promise<string[]> {
+    const results = await execCmdAndParseJson(
+      `helm list --filter '${WarpRouteMonitorHelmManager.helmReleasePrefix}.+' -o json -n ${namespace}`,
+    );
+    return results.map((r: any) => r.name);
+  }
+
+  // This method is used to uninstall any stale Warp Monitors.
+  // This can happen if a Warp Route ID is changed or removed.
+  // Any warp monitor helm releases found that do not relate to known warp route ids
+  // will be prompted for uninstallation.
+  static async uninstallUnknownWarpMonitorReleases(namespace: string) {
+    const allExpectedHelmReleaseNames = Object.values(WarpRouteIds).map(
+      WarpRouteMonitorHelmManager.getHelmReleaseName,
+    );
+    const helmReleases =
+      await WarpRouteMonitorHelmManager.getWarpMonitorHelmReleases(namespace);
+
+    const unknownHelmReleases = difference(
+      new Set(helmReleases),
+      new Set(allExpectedHelmReleaseNames),
+    );
+    for (const helmRelease of unknownHelmReleases) {
+      console.log(
+        `Unknown Warp Monitor Helm Release: ${helmRelease} (possibly a release from a stale Warp Route ID).`,
+      );
+      const uninstall = await confirm({
+        message:
+          "Would you like to uninstall this Helm Release? Make extra sure it shouldn't exist!",
+      });
+      if (uninstall) {
+        console.log(`Uninstalling Helm Release: ${helmRelease}`);
+        await removeHelmRelease(helmRelease, namespace);
+      } else {
+        console.log(`Skipping uninstall of Helm Release: ${helmRelease}`);
+      }
+    }
+  }
 }

typescript/sdk/src/metadata/matchingList.ts

@@ -15,6 +15,7 @@ const DomainSchema = z.union([
 const AddressSchema = z.union([z.literal('*'), ZHash, z.array(ZHash)]);
 
 const MatchingListElementSchema = z.object({
+  messageId: AddressSchema.optional(),
   originDomain: DomainSchema.optional(),
   senderAddress: AddressSchema.optional(),
   destinationDomain: DomainSchema.optional(),