feat: remove GCP storage dependency from SDK

.changeset/many-clouds-bow.md

@@ -0,0 +1,5 @@
+---
+'@hyperlane-xyz/sdk': minor
+---
+
+Call google storage API directly and remove @google-cloud/storage dependency from the SDK.

typescript/infra/scripts/check/check-validator-version.ts

@@ -28,6 +28,8 @@ const acceptableValidatorVersions: Record<string, string> = {
   'd834d8147628584acd78a81e344bff76472d707e': 'nov-21-bsquared',
   // Nov 21 swell/lumiaprism deploy
   'b35c105f197267072daa14bb3d83c62410b96fac': 'nov-21-swell',
+  // Dec 4 deploy
+  'a7f3967e047c2c5aabb8cc442e4acad435fa32ab': 'dec-4-batch',
   // Rolled out only to AW infra before 1.0.0, just 1 commit behind 1.0.0
   'a64af8be9a76120d0cfc727bb70660fa07e70cce': 'pre-1.0.0',
   // 1.0.0

typescript/sdk/package.json

@@ -8,7 +8,6 @@
     "@chain-registry/types": "^0.50.14",
     "@cosmjs/cosmwasm-stargate": "^0.32.4",
     "@cosmjs/stargate": "^0.32.4",
-    "@google-cloud/storage": "7.14.0",
     "@hyperlane-xyz/core": "5.8.3",
     "@hyperlane-xyz/utils": "7.3.0",
     "@safe-global/api-kit": "1.3.0",

typescript/sdk/src/gcp/storage.ts

@@ -1,5 +1,3 @@
-import { Storage } from '@google-cloud/storage';
-
 export const GCS_BUCKET_REGEX =
   /^(?:(?:https?:\/\/)?([^/]+)\.storage\.googleapis\.com\/?|gs:\/\/([^/]+))$/;
 
@@ -18,66 +16,91 @@ export interface StorageConfig {
 }
 
 export class GcpStorageWrapper {
-  private readonly client: Storage;
   private readonly bucket: string;
   private cache: Record<string, StorageReceipt<any>> | undefined;
+  private readonly baseUrl: string;
 
   static fromBucketUrl(bucketUrl: string): GcpStorageWrapper {
     const match = bucketUrl.match(GCS_BUCKET_REGEX);
     if (!match) throw new Error('Could not parse bucket url');
     return new GcpStorageWrapper({
-      bucket: match[1],
+      // Handle both http and gs:// formats
+      bucket: match[1] || match[2],
       caching: true,
     });
   }
 
   constructor(readonly config: StorageConfig) {
-    this.client = new Storage({
-      projectId: config.projectId,
-      keyFilename: config.keyFilename,
-    });
     this.bucket = config.bucket;
+    this.baseUrl = `https://storage.googleapis.com/storage/v1/b/${this.bucket}/o`;
     if (config.caching) {
       this.cache = {};
     }
   }
 
-  formatKey(key: string): string {
+  private formatKey(key: string): string {
     return this.config.folder ? `${this.config.folder}/${key}` : key;
   }
 
-  async getObject<T>(key: string): Promise<StorageReceipt<T> | undefined> {
-    const Key = this.formatKey(key);
-    if (this.cache?.[Key]) {
-      return this.cache![Key];
+  private getCachedObject<T>(key: string): StorageReceipt<T> | undefined {
+    return this.cache?.[key];
+  }
+
+  private setCachedObject<T>(key: string, value: StorageReceipt<T>): void {
+    if (this.cache) {
+      this.cache[key] = value;
     }
+  }
 
-    try {
-      const bucket = this.client.bucket(this.bucket);
-      const file = bucket.file(Key);
-      const [exists] = await file.exists();
+  private async fetchMetadata(key: string): Promise<any> {
+    const url = `${this.baseUrl}/${encodeURIComponent(key)}`;
+    const response = await fetch(url);
 
-      if (!exists) {
-        return undefined;
-      }
+    if (response.status === 404) return undefined;
+
+    if (!response.ok) {
+      throw new Error(
+        `Failed to fetch object metadata: ${response.statusText}`,
+      );
+    }
+
+    return response.json();
+  }
+
+  private async fetchContent(key: string): Promise<string> {
+    const url = `${this.baseUrl}/${encodeURIComponent(key)}?alt=media`;
+    const response = await fetch(url);
+
+    if (!response.ok) {
+      throw new Error(`Failed to fetch object content: ${response.statusText}`);
+    }
+
+    return response.text();
+  }
 
-      const [metadata] = await file.getMetadata();
-      const [contents] = await file.download();
-      const body = contents.toString('utf-8');
+  async getObject<T>(key: string): Promise<StorageReceipt<T> | undefined> {
+    const formattedKey = this.formatKey(key);
+    const cachedObject = this.getCachedObject<T>(formattedKey);
+    if (cachedObject) {
+      return cachedObject;
+    }
 
+    try {
+      const metadata = await this.fetchMetadata(formattedKey);
+      if (!metadata) return undefined;
+
+      const body = await this.fetchContent(formattedKey);
       const result = {
         data: JSON.parse(body),
         // If no updated date is provided, use the Unix epoch start
         // 0 = Unix epoch start (1970-01-01T00:00:00.000Z)
         modified: new Date(metadata.updated ?? 0),
       };
 
-      if (this.cache) {
-        this.cache[Key] = result;
-      }
+      this.setCachedObject(formattedKey, result);
       return result;
     } catch (e: any) {
-      if (e.code === 404) {
+      if (e.status === 404) {
         return undefined;
       }
       throw e;

yarn.lock

@@ -7205,30 +7205,6 @@ __metadata:
   languageName: node
   linkType: hard
 
-"@google-cloud/paginator@npm:^5.0.0":
-  version: 5.0.2
-  resolution: "@google-cloud/paginator@npm:5.0.2"
-  dependencies:
-    arrify: "npm:^2.0.0"
-    extend: "npm:^3.0.2"
-  checksum: 10/b64ba2029b77fdcf3c827aea0b6d128122fd1d2f4aa8c1ba70747cba0659d4216a283769fb3bbeb8f726176f5282624637f02c30f118a010e05838411da0cb76
-  languageName: node
-  linkType: hard
-
-"@google-cloud/projectify@npm:^4.0.0":
-  version: 4.0.0
-  resolution: "@google-cloud/projectify@npm:4.0.0"
-  checksum: 10/fdccdda0b50855c35541d71c46a6603f3302ff1a00108d946272cb2167435da00e2a2da5963fe489f4f5a4a9eb6320abeb97d3269974a972ae89f5df8451922d
-  languageName: node
-  linkType: hard
-
-"@google-cloud/promisify@npm:^4.0.0":
-  version: 4.0.0
-  resolution: "@google-cloud/promisify@npm:4.0.0"
-  checksum: 10/c5de81321b3a5c567edcbe0b941fb32644611147f3ba22f20575918c225a979988a99bc2ebda05ac914fa8714b0a54c69be72c3f46c7a64c3b19db7d7fba8d04
-  languageName: node
-  linkType: hard
-
 "@google-cloud/secret-manager@npm:^5.5.0":
   version: 5.5.0
   resolution: "@google-cloud/secret-manager@npm:5.5.0"
@@ -7238,29 +7214,6 @@ __metadata:
   languageName: node
   linkType: hard
 
-"@google-cloud/storage@npm:7.14.0":
-  version: 7.14.0
-  resolution: "@google-cloud/storage@npm:7.14.0"
-  dependencies:
-    "@google-cloud/paginator": "npm:^5.0.0"
-    "@google-cloud/projectify": "npm:^4.0.0"
-    "@google-cloud/promisify": "npm:^4.0.0"
-    abort-controller: "npm:^3.0.0"
-    async-retry: "npm:^1.3.3"
-    duplexify: "npm:^4.1.3"
-    fast-xml-parser: "npm:^4.4.1"
-    gaxios: "npm:^6.0.2"
-    google-auth-library: "npm:^9.6.3"
-    html-entities: "npm:^2.5.2"
-    mime: "npm:^3.0.0"
-    p-limit: "npm:^3.0.1"
-    retry-request: "npm:^7.0.0"
-    teeny-request: "npm:^9.0.0"
-    uuid: "npm:^8.0.0"
-  checksum: 10/0726fde2697da696637fab91ebd756354a58c1331f6a0b9ecc5011de4aae72cd9e1fe3e9564aee15c6a2118e45ed0ae8c3ac9685c6581db6107080f906a949e9
-  languageName: node
-  linkType: hard
-
 "@grpc/grpc-js@npm:~1.10.3":
   version: 1.10.8
   resolution: "@grpc/grpc-js@npm:1.10.8"
@@ -7602,7 +7555,6 @@ __metadata:
     "@cosmjs/cosmwasm-stargate": "npm:^0.32.4"
     "@cosmjs/stargate": "npm:^0.32.4"
     "@eslint/js": "npm:^9.15.0"
-    "@google-cloud/storage": "npm:7.14.0"
     "@hyperlane-xyz/core": "npm:5.8.3"
     "@hyperlane-xyz/utils": "npm:7.3.0"
     "@nomiclabs/hardhat-ethers": "npm:^2.2.3"
@@ -17874,13 +17826,6 @@ __metadata:
   languageName: node
   linkType: hard
 
-"arrify@npm:^2.0.0":
-  version: 2.0.1
-  resolution: "arrify@npm:2.0.1"
-  checksum: 10/067c4c1afd182806a82e4c1cb8acee16ab8b5284fbca1ce29408e6e91281c36bb5b612f6ddfbd40a0f7a7e0c75bf2696eb94c027f6e328d6e9c52465c98e4209
-  languageName: node
-  linkType: hard
-
 "as-table@npm:^1.0.36":
   version: 1.0.55
   resolution: "as-table@npm:1.0.55"
@@ -21074,7 +21019,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"duplexify@npm:^4.0.0, duplexify@npm:^4.1.2, duplexify@npm:^4.1.3":
+"duplexify@npm:^4.0.0, duplexify@npm:^4.1.2":
   version: 4.1.3
   resolution: "duplexify@npm:4.1.3"
   dependencies:
@@ -23202,17 +23147,6 @@ __metadata:
   languageName: node
   linkType: hard
 
-"fast-xml-parser@npm:^4.4.1":
-  version: 4.5.0
-  resolution: "fast-xml-parser@npm:4.5.0"
-  dependencies:
-    strnum: "npm:^1.0.5"
-  bin:
-    fxparser: src/cli/cli.js
-  checksum: 10/dc9571c10e7b57b5be54bcd2d92f50c446eb42ea5df347d253e94dd14eb99b5300a6d172e840f151e0721933ca2406165a8d9b316a6d777bf0596dc4fe1df756
-  languageName: node
-  linkType: hard
-
 "fastq@npm:^1.6.0":
   version: 1.13.0
   resolution: "fastq@npm:1.13.0"
@@ -23955,19 +23889,6 @@ __metadata:
   languageName: node
   linkType: hard
 
-"gaxios@npm:^6.0.2":
-  version: 6.7.1
-  resolution: "gaxios@npm:6.7.1"
-  dependencies:
-    extend: "npm:^3.0.2"
-    https-proxy-agent: "npm:^7.0.1"
-    is-stream: "npm:^2.0.0"
-    node-fetch: "npm:^2.6.9"
-    uuid: "npm:^9.0.1"
-  checksum: 10/c85599162208884eadee91215ebbfa1faa412551df4044626cb561300e15193726e8f23d63b486533e066dadad130f58ed872a23acab455238d8d48b531a0695
-  languageName: node
-  linkType: hard
-
 "gcp-metadata@npm:^6.1.0":
   version: 6.1.0
   resolution: "gcp-metadata@npm:6.1.0"
@@ -24475,20 +24396,6 @@ __metadata:
   languageName: node
   linkType: hard
 
-"google-auth-library@npm:^9.6.3":
-  version: 9.15.0
-  resolution: "google-auth-library@npm:9.15.0"
-  dependencies:
-    base64-js: "npm:^1.3.0"
-    ecdsa-sig-formatter: "npm:^1.0.11"
-    gaxios: "npm:^6.1.1"
-    gcp-metadata: "npm:^6.1.0"
-    gtoken: "npm:^7.0.0"
-    jws: "npm:^4.0.0"
-  checksum: 10/fba2db9732bbf1b3a3a2e2b45131ba8e8aba297377f1c104d0b2ab3386bbc1e02047f20b8a7afca1c6308492da1540104618f1c7b5cd539703552e10399c560e
-  languageName: node
-  linkType: hard
-
 "google-gax@npm:^4.0.3":
   version: 4.3.3
   resolution: "google-gax@npm:4.3.3"
@@ -25170,13 +25077,6 @@ __metadata:
   languageName: node
   linkType: hard
 
-"html-entities@npm:^2.5.2":
-  version: 2.5.2
-  resolution: "html-entities@npm:2.5.2"
-  checksum: 10/4ec12ebdf2d5ba8192c68e1aef3c1e4a4f36b29246a0a88464fe278a54517d0196d3489af46a3145c7ecacb4fc5fd50497be19eb713b810acab3f0efcf36fdc2
-  languageName: node
-  linkType: hard
-
 "html-escaper@npm:^2.0.0":
   version: 2.0.2
   resolution: "html-escaper@npm:2.0.2"
@@ -30113,7 +30013,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"p-limit@npm:^3.0.1, p-limit@npm:^3.0.2, p-limit@npm:^3.1.0":
+"p-limit@npm:^3.0.2, p-limit@npm:^3.1.0":
   version: 3.1.0
   resolution: "p-limit@npm:3.1.0"
   dependencies:
@@ -36411,7 +36311,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"uuid@npm:^8.0.0, uuid@npm:^8.3.2":
+"uuid@npm:^8.3.2":
   version: 8.3.2
   resolution: "uuid@npm:8.3.2"
   bin: