api: delete remaining uses of Shovel

daimo-eth · Sep 1, 2024 · 717c120 · 717c120
1 parent d8abfb8
commit 717c120
Show file tree

Hide file tree

Showing 10 changed files with 195 additions and 319 deletions.
diff --git a/packages/daimo-api/src/contract/foreignCoinIndexer.ts b/packages/daimo-api/src/contract/foreignCoinIndexer.ts
@@ -12,7 +12,6 @@ import {
 } from "@daimo/common";
 import { zeroAddr } from "@daimo/contract";
 import { Kysely } from "kysely";
-import { Pool } from "pg";
 import { Address, Hex, bytesToHex, getAddress } from "viem";
 
 import { Transfer } from "./homeCoinIndexer";
@@ -60,7 +59,7 @@ export class ForeignCoinIndexer extends Indexer {
     super("FOREIGN-COIN");
   }
 
-  async load(pg: Pool, kdb: Kysely<IndexDB>, from: number, to: number) {
+  async load(kdb: Kysely<IndexDB>, from: number, to: number) {
     const startMs = performance.now();
 
     const result = await retryBackoff(

diff --git a/packages/daimo-api/src/contract/homeCoinIndexer.ts b/packages/daimo-api/src/contract/homeCoinIndexer.ts
@@ -11,7 +11,6 @@ import {
 } from "@daimo/common";
 import { DaimoNonce } from "@daimo/userop";
 import { Kysely } from "kysely";
-import { Pool } from "pg";
 import { Address, Hex, bytesToHex, getAddress, numberToHex } from "viem";
 
 import { ForeignCoinIndexer } from "./foreignCoinIndexer";
@@ -62,7 +61,7 @@ export class HomeCoinIndexer extends Indexer {
     return { numTransfers: this.allTransfers.length };
   }
 
-  async load(pg: Pool, kdb: Kysely<IndexDB>, from: number, to: number) {
+  async load(kdb: Kysely<IndexDB>, from: number, to: number) {
     const startTime = Date.now();
 
     const result = await retryBackoff(

diff --git a/packages/daimo-api/src/contract/indexer.ts b/packages/daimo-api/src/contract/indexer.ts
@@ -1,5 +1,4 @@
 import { Kysely } from "kysely";
-import { Pool } from "pg";
 
 import { DB as IndexDB } from "../codegen/dbIndex";
 
@@ -14,7 +13,6 @@ export abstract class Indexer {
 
   // Loads a batch of blocks from the database.
   public abstract load(
-    pg: Pool,
     kdb: Kysely<IndexDB>,
     from: number,
     to: number

diff --git a/packages/daimo-api/src/contract/keyRegistry.ts b/packages/daimo-api/src/contract/keyRegistry.ts
@@ -1,11 +1,12 @@
 import {
   KeyData,
   assert,
+  assertNotNull,
   contractFriendlyKeyToDER,
+  debugJson,
   retryBackoff,
 } from "@daimo/common";
 import { Kysely } from "kysely";
-import { Pool } from "pg";
 import { Address, Hex, bytesToHex, getAddress } from "viem";
 
 import { Indexer } from "./indexer";
@@ -19,7 +20,6 @@ export interface KeyChange {
   transactionHash: Hex;
   logIndex: number;
   address: Address;
-  account: Address;
   keySlot: number;
   key: [Hex, Hex];
 }
@@ -41,27 +41,49 @@ export class KeyRegistry extends Indexer {
     this.listeners.push(listener);
   }
 
-  async load(pg: Pool, kdb: Kysely<IndexDB>, from: number, to: number) {
+  async load(kdb: Kysely<IndexDB>, from: number, to: number) {
     const startTime = Date.now();
-    const changes: KeyChange[] = [];
-    changes.push(...(await this.loadKeyChange(pg, from, to, "added")));
-    changes.push(...(await this.loadKeyChange(pg, from, to, "removed")));
+
+    const rows = await retryBackoff(`keyRegistry-${from}-${to}`, () =>
+      kdb
+        .selectFrom("index.daimo_acct_update")
+        .selectAll()
+        .where("chain_id", "=", "" + chainConfig.chainL2.id)
+        .where((eb) => eb.between("block_num", "" + from, "" + to))
+        .orderBy("block_num")
+        .orderBy("tx_idx")
+        .orderBy("log_idx")
+        .execute()
+    );
 
     if (this.updateLastProcessedCheckStale(from, to)) return;
 
-    changes!.sort((a, b) => {
-      const bdiff = a.blockNumber - b.blockNumber;
-      if (bdiff !== 0n) return Number(bdiff);
-      const tdiff = a.transactionIndex - b.transactionIndex;
-      if (tdiff !== 0) return tdiff;
-      return a.logIndex - b.logIndex;
-    });
+    const changes: KeyChange[] = rows
+      .filter((row) => row.key_slot != null)
+      .map((row) => ({
+        change: (function () {
+          if (row.log_name === "SigningKeyAdded") return "added";
+          else if (row.log_name === "SigningKeyRemoved") return "removed";
+          else throw new Error(`Unexpected key log: ${debugJson(row)}`);
+        })(),
+        address: getAddress(bytesToHex(row.log_addr)),
+        blockNumber: BigInt(row.block_num),
+        key: (function () {
+          const k = assertNotNull(row.key);
+          if (k.length !== 64) throw new Error(`Bad key: ${debugJson(row)}`);
+          return [bytesToHex(k.subarray(0, 32)), bytesToHex(k.subarray(32))];
+        })(),
+        keySlot: assertNotNull(row.key_slot),
+        logIndex: Number(row.log_idx),
+        transactionHash: bytesToHex(row.tx_hash),
+        transactionIndex: Number(row.tx_idx),
+      }));
+
     for (const change of changes) {
-      const addr = getAddress(change.address);
-      if (this.addrToLogs.get(addr) === undefined) {
-        this.addrToLogs.set(addr, []);
-      }
-      this.addrToLogs.get(addr)!.push(change);
+      const addr = change.address;
+      const addrLogs = this.addrToLogs.get(addr) || [];
+      addrLogs.push(change);
+      this.addrToLogs.set(addr, addrLogs);
 
       if (!change.key) throw new Error("[KEY-REG] Invalid event, no key");
       const slot = change.keySlot;
@@ -95,6 +117,9 @@ export class KeyRegistry extends Indexer {
           this.keyToAddr.delete(derKey);
           break;
         }
+        default: {
+          throw new Error(`Invalid KeyChange: ${debugJson(change)}`);
+        }
       }
     }
     if (changes.length === 0) return;
@@ -107,54 +132,6 @@ export class KeyRegistry extends Indexer {
     this.listeners.forEach((l) => l(changes));
   }
 
-  private async loadKeyChange(
-    pg: Pool,
-    from: number,
-    to: number,
-    change: "added" | "removed"
-  ): Promise<KeyChange[]> {
-    let table: string = "";
-    if (change === "added") {
-      table = "key_added";
-    } else if (change === "removed") {
-      table = "key_removed";
-    } else {
-      throw new Error(`Invalid key change ${change}`);
-    }
-    const result = await retryBackoff(`keyRegistry-logs-query-${table}`, () =>
-      pg.query(
-        `
-        select
-          block_num,
-          tx_idx,
-          tx_hash,
-          log_idx,
-          log_addr,
-          account,
-          key_slot,
-          array_agg(key order by abi_idx asc) as key
-        from ${table}
-        where block_num >= $1 and block_num <= $2
-        and chain_id = $3
-        group by block_num, tx_idx, tx_hash, log_idx, log_addr, account, key_slot
-      `,
-        [from, to, chainConfig.chainL2.id]
-      )
-    );
-
-    return result.rows.map((row: any) => ({
-      change,
-      blockNumber: BigInt(row.block_num),
-      transactionIndex: row.tx_idx,
-      transactionHash: bytesToHex(row.tx_hash, { size: 32 }),
-      logIndex: row.log_idx,
-      address: getAddress(bytesToHex(row.log_addr, { size: 20 })),
-      account: getAddress(bytesToHex(row.account, { size: 20 })),
-      keySlot: row.key_slot,
-      key: row.key.map((k: Buffer) => bytesToHex(k)) as [Hex, Hex],
-    }));
-  }
-
   /** Find address by DER key */
   async resolveKey(key: Hex): Promise<Address | null> {
     return this.keyToAddr.get(key) || null;

diff --git a/packages/daimo-api/src/contract/nameRegistry.ts b/packages/daimo-api/src/contract/nameRegistry.ts
@@ -5,15 +5,13 @@ import {
   EAccount,
   assertEqual,
   assertNotNull,
-  guessTimestampFromNum,
   isValidName,
   now,
-  validateName,
   retryBackoff,
+  validateName,
 } from "@daimo/common";
 import { nameRegistryProxyConfig, teamDaimoFaucetAddr } from "@daimo/contract";
 import { Kysely } from "kysely";
-import { Pool } from "pg";
 import {
   Address,
   bytesToHex,
@@ -109,34 +107,35 @@ export class NameRegistry extends Indexer {
     return { numAccounts: this.accounts.length, numLogs: this.logs.length };
   }
 
-  async load(pg: Pool, kdb: Kysely<IndexDB>, from: number, to: number) {
-    const startTime = Date.now();
-    const result = await retryBackoff(
+  async load(kdb: Kysely<IndexDB>, from: number, to: number) {
+    const startMs = performance.now();
+    const rows = await retryBackoff(
       `nameRegistry-logs-query-${from}-${to}`,
       () =>
-        pg.query(
-          `select block_num, addr, name
-          from names
-          where block_num >= $1
-          and block_num <= $2
-          and chain_id = $3`,
-          [from, to, chainConfig.chainL2.id]
-        )
+        kdb
+          .selectFrom("index.daimo_name")
+          .selectAll()
+          .where("chain_id", "=", "" + chainConfig.chainL2.id)
+          .where((eb) => eb.between("block_num", "" + from, "" + to))
+          .orderBy("block_num")
+          .orderBy("tx_idx")
+          .orderBy("log_idx")
+          .execute()
     );
 
     if (this.updateLastProcessedCheckStale(from, to)) return;
 
-    const names = result.rows.map((r: any) => {
+    const names = rows.map((r) => {
       return {
-        timestamp: guessTimestampFromNum(r.block_num, chainConfig.daimoChain),
+        timestamp: Number(r.block_ts),
         name: bytesToString(r.name, { size: 32 }),
         addr: getAddress(bytesToHex(r.addr, { size: 20 })),
       };
     });
     this.logs.push(...names);
     names.forEach(this.cacheAccount);
 
-    const elapsedMs = (Date.now() - startTime) | 0;
+    const elapsedMs = (performance.now() - startMs) | 0;
     console.log(`[NAME-REG] loaded ${names.length} names in ${elapsedMs}ms`);
   }
 

diff --git a/packages/daimo-api/src/contract/noteIndexer.ts b/packages/daimo-api/src/contract/noteIndexer.ts
@@ -9,7 +9,6 @@ import {
   retryBackoff,
 } from "@daimo/common";
 import { Kysely } from "kysely";
-import { Pool } from "pg";
 import { Address, Hex, bytesToHex, getAddress } from "viem";
 
 import { Indexer } from "./indexer";
@@ -18,7 +17,7 @@ import { OpIndexer } from "./opIndexer";
 import { DB as IndexDB } from "../codegen/dbIndex";
 import { chainConfig } from "../env";
 import { PaymentMemoTracker } from "../offchain/paymentMemoTracker";
-import { senderIdKey, logCoordinateKey } from "../utils/indexing";
+import { logCoordinateKey, senderIdKey } from "../utils/indexing";
 
 interface NoteLog {
   blockNum: number;
@@ -55,13 +54,13 @@ export class NoteIndexer extends Indexer {
     super("NOTE");
   }
 
-  async load(pg: Pool, kdb: Kysely<IndexDB>, from: number, to: number) {
+  async load(kdb: Kysely<IndexDB>, from: number, to: number) {
     // Load notes contract event logs
-    const startMs = Date.now();
-    const logs = await this.loadNoteLogs(pg, from, to);
+    const startMs = performance.now();
+    const logs = await this.loadNoteLogs(kdb, from, to);
     if (logs.length === 0) return;
 
-    const elapsedMs = (Date.now() - startMs) | 0;
+    const elapsedMs = (performance.now() - startMs) | 0;
     console.log(`[NOTE] ${elapsedMs}ms: loaded ${logs.length} logs`);
 
     if (this.updateLastProcessedCheckStale(from, to)) return;
@@ -78,52 +77,35 @@ export class NoteIndexer extends Indexer {
   }
 
   private async loadNoteLogs(
-    pg: Pool,
+    kdb: Kysely<IndexDB>,
     from: number,
     to: number
   ): Promise<NoteLog[]> {
-    const result = await retryBackoff(
+    const rows = await retryBackoff(
       `noteIndexer-logs-query-${from}-${to}`,
       () =>
-        pg.query(
-          `
-      select * from (
-        select
-          block_num,
-          tx_idx,
-          log_idx,
-          tx_hash,
-          f,
-          null as redeemer,
-          ephemeral_owner,
-          amount,
-          log_addr
-        from note_created
-        where block_num >= $1
-        and block_num <= $2
-        and chain_id = $3
-        union 
-        select
-          block_num,
-          tx_idx,
-          log_idx,
-          tx_hash,
-          f,
-          redeemer,
-          ephemeral_owner,
-          amount,
-          log_addr
-        from note_redeemed
-        where block_num >= $1
-        and block_num <= $2
-        and chain_id = $3
-      ) as notelogs
-      order by block_num asc, tx_idx asc, log_idx asc
-    `,
-          [from, to, chainConfig.chainL2.id]
-        )
+        kdb
+          .selectFrom("index.daimo_note")
+          .selectAll()
+          .where("chain_id", "=", "" + chainConfig.chainL2.id)
+          .where((eb) => eb.between("block_num", "" + from, "" + to))
+          .orderBy("block_num")
+          .orderBy("tx_idx")
+          .orderBy("log_idx")
+          .execute()
     );
-    return result.rows.map(rowToNoteLog);
+
+    return rows.map((r) => ({
+      blockNum: Number(r.block_num),
+      transactionIndex: Number(r.tx_idx),
+      logIndex: Number(r.log_idx),
+      transactionHash: bytesToHex(r.tx_hash, { size: 32 }),
+      from: getAddress(bytesToHex(r.creator, { size: 20 })),
+      redeemer: r.redeemer && getAddress(bytesToHex(r.redeemer, { size: 20 })),
+      ephemeralOwner: getAddress(bytesToHex(r.ephemeral_owner, { size: 20 })),
+      amount: BigInt(r.amount),
+      logAddr: getAddress(bytesToHex(r.log_addr, { size: 20 })),
+    }));
   }
 
   async handleNoteLogs(logs: NoteLog[]): Promise<DaimoNoteStatus[]> {
@@ -258,17 +240,3 @@ export class NoteIndexer extends Indexer {
     return this.noteLogs.get(ephemeralOwner)?.claim;
   }
 }
-
-function rowToNoteLog(r: any): NoteLog {
-  return {
-    blockNum: r.block_num,
-    transactionIndex: r.tx_idx,
-    logIndex: r.log_idx,
-    transactionHash: bytesToHex(r.tx_hash, { size: 32 }),
-    from: getAddress(bytesToHex(r.f, { size: 20 })),
-    redeemer: r.redeemer && getAddress(bytesToHex(r.redeemer, { size: 20 })),
-    ephemeralOwner: getAddress(bytesToHex(r.ephemeral_owner, { size: 20 })),
-    amount: BigInt(r.amount),
-    logAddr: getAddress(bytesToHex(r.log_addr, { size: 20 })),
-  };
-}