fix: optimize localDB getAllRecords performance

apps/desktop/App.tsx

@@ -12,3 +12,4 @@ import {
 initSentry();
 
 export default withSentryHOC(KitProvider);
+// export default KitProvider;

development/spellCheckerSkipWords.js

@@ -20,6 +20,7 @@ module.exports = [
   'hdk',
   'dkey',
   'impls',
+  'ttl',
   'Sollet',
   'Solflare',
   'encryptors',

packages/kit-bg/src/dbs/local/LocalDbBase.ts

@@ -91,7 +91,6 @@ import type {
   IDBDeviceSettings,
   IDBEnsureAccountNameNotDuplicateParams,
   IDBExternalAccount,
-  IDBGetAllWalletsParams,
   IDBGetWalletsParams,
   IDBIndexedAccount,
   IDBRemoveWalletParams,
@@ -511,30 +510,6 @@ export abstract class LocalDbBase extends LocalDbBaseContainer {
   walletSortFn = (a: IDBWallet, b: IDBWallet) =>
     (a.walletOrder ?? 0) - (b.walletOrder ?? 0);
 
-  async getAllWallets({
-    refillWalletInfo,
-  }: IDBGetAllWalletsParams = {}): Promise<{
-    wallets: IDBWallet[];
-  }> {
-    let { records } = await this.getAllRecords({
-      name: ELocalDBStoreNames.Wallet,
-    });
-    if (refillWalletInfo) {
-      const { devices: allDevices } = await this.getAllDevices();
-      const refilledWalletsCache: {
-        [walletId: string]: IDBWallet;
-      } = {};
-      records = await Promise.all(
-        records.map((wallet) =>
-          this.refillWalletInfo({ wallet, refilledWalletsCache, allDevices }),
-        ),
-      );
-    }
-    return {
-      wallets: records,
-    };
-  }
-
   // eslint-disable-next-line spellcheck/spell-checker
   /**
    * Get wallets
@@ -552,8 +527,9 @@ export abstract class LocalDbBase extends LocalDbBaseContainer {
     let allIndexedAccounts: IDBIndexedAccount[] | undefined;
     if (includingAccounts) {
       if (!allIndexedAccounts) {
-        allIndexedAccounts = (await this.getAllIndexedAccounts())
-          .indexedAccounts;
+        allIndexedAccounts =
+          option?.allIndexedAccounts ||
+          (await this.getAllIndexedAccounts()).indexedAccounts;
       }
     }
 
@@ -577,8 +553,9 @@ export abstract class LocalDbBase extends LocalDbBaseContainer {
     };
 
     // get all wallets for account selector
-    let { wallets } = await this.getAllWallets();
-    const { devices: allDevices } = await this.getAllDevices();
+    let wallets = option?.allWallets || (await this.getAllWallets()).wallets;
+    const allDevices =
+      option?.allDevices || (await this.getAllDevices()).devices;
     const hiddenWalletsMap: Partial<{
       [dbDeviceId: string]: IDBWallet[];
     }> = {};
@@ -2611,7 +2588,8 @@ export abstract class LocalDbBase extends LocalDbBaseContainer {
     accounts: IDBAccount[];
   }> {
     const wallet = await this.getWalletSafe({ walletId });
-    if (!wallet) {
+    if (!wallet || !wallet?.accounts?.length) {
+      // if (!wallet) {
       return { accounts: [] };
     }
     const { accounts } = await this.getAllAccounts({
@@ -2648,13 +2626,14 @@ export abstract class LocalDbBase extends LocalDbBaseContainer {
     const indexedAccount = await this.getIndexedAccount({
       id: indexedAccountId,
     });
-    const { accounts } = await this.getAllAccounts();
-    return accounts
+    const allDbAccounts = (await this.getAllAccounts()).accounts;
+    const accounts = allDbAccounts
       .filter(
         (account) =>
           account.indexedAccountId === indexedAccountId && indexedAccountId,
       )
       .map((account) => this.refillAccountInfo({ account, indexedAccount }));
+    return { accounts, allDbAccounts };
   }
 
   async getAccount({ accountId }: { accountId: string }): Promise<IDBAccount> {
@@ -2749,18 +2728,101 @@ export abstract class LocalDbBase extends LocalDbBaseContainer {
     });
   }
 
-  async getAllIndexedAccounts() {
+  async getAllDevices(): Promise<{ devices: IDBDevice[] }> {
+    const cacheKey = 'allDbDevices';
+    const allDevicesInCache = this.dbAllRecordsCache.get(
+      cacheKey,
+    ) as IDBDevice[];
+    if (allDevicesInCache && allDevicesInCache.length) {
+      return { devices: allDevicesInCache };
+    }
+    const { records: devices } = await this.getAllRecords({
+      name: ELocalDBStoreNames.Device,
+    });
+    devices.forEach((item) => this.refillDeviceInfo({ device: item }));
+    this.dbAllRecordsCache.set(cacheKey, devices);
+    return { devices };
+  }
+
+  async getAllWallets(): Promise<{
+    wallets: IDBWallet[];
+  }> {
+    const cacheKey = 'allDbWallets';
+    const allWalletsInCache = this.dbAllRecordsCache.get(
+      cacheKey,
+    ) as IDBWallet[];
+    if (allWalletsInCache && allWalletsInCache.length) {
+      return { wallets: allWalletsInCache };
+    }
+    const { records: wallets } = await this.getAllRecords({
+      name: ELocalDBStoreNames.Wallet,
+    });
+    this.dbAllRecordsCache.set(cacheKey, wallets);
+    return {
+      wallets,
+    };
+  }
+
+  // async getAllWallets({
+  //   refillWalletInfo,
+  // }: IDBGetAllWalletsParams = {}): Promise<{
+  //   wallets: IDBWallet[];
+  // }> {
+  //   let { records } = await this.getAllRecords({
+  //     name: ELocalDBStoreNames.Wallet,
+  //   });
+  //   if (refillWalletInfo) {
+  //     const { devices: allDevices } = await this.getAllDevices();
+  //     const refilledWalletsCache: {
+  //       [walletId: string]: IDBWallet;
+  //     } = {};
+  //     records = await Promise.all(
+  //       records.map((wallet) =>
+  //         this.refillWalletInfo({ wallet, refilledWalletsCache, allDevices }),
+  //       ),
+  //     );
+  //   }
+  //   return {
+  //     wallets: records,
+  //   };
+  // }
+
+  async getAllIndexedAccounts(): Promise<{
+    indexedAccounts: IDBIndexedAccount[];
+  }> {
+    const cacheKey = 'allDbIndexedAccounts';
+    const allIndexedAccountsInCache = this.dbAllRecordsCache.get(
+      cacheKey,
+    ) as IDBIndexedAccount[];
+    if (allIndexedAccountsInCache && allIndexedAccountsInCache.length) {
+      return { indexedAccounts: allIndexedAccountsInCache };
+    }
     const { records: indexedAccounts } = await this.getAllRecords({
       name: ELocalDBStoreNames.IndexedAccount,
     });
+    this.dbAllRecordsCache.set(cacheKey, indexedAccounts);
     return { indexedAccounts };
   }
 
-  async getAllAccounts({ ids }: { ids?: string[] } = {}) {
+  async getAllAccounts({ ids }: { ids?: string[] } = {}): Promise<{
+    accounts: IDBAccount[];
+  }> {
+    const cacheKey = 'allDbAccounts';
+    if (!ids) {
+      const allDbAccountsInCache = this.dbAllRecordsCache.get(
+        cacheKey,
+      ) as IDBAccount[];
+      if (allDbAccountsInCache && allDbAccountsInCache?.length) {
+        return { accounts: allDbAccountsInCache };
+      }
+    }
     const { records: accounts } = await this.getAllRecords({
       name: ELocalDBStoreNames.Account,
       ids,
     });
+    if (!ids) {
+      this.dbAllRecordsCache.set(cacheKey, accounts);
+    }
     return { accounts };
   }
 
@@ -2961,9 +3023,11 @@ export abstract class LocalDbBase extends LocalDbBaseContainer {
 
       if (params.indexedAccountId) {
         // TODO low performance
-        accounts = await this.getAccountsInSameIndexedAccountId({
-          indexedAccountId: params.indexedAccountId,
-        });
+        accounts = (
+          await this.getAccountsInSameIndexedAccountId({
+            indexedAccountId: params.indexedAccountId,
+          })
+        ).accounts;
       }
       if (params.accountId) {
         const account = await this.getAccountSafe({
@@ -3031,15 +3095,6 @@ export abstract class LocalDbBase extends LocalDbBaseContainer {
 
   // ---------------------------------------------- device
 
-  async getAllDevices(): Promise<{ devices: IDBDevice[] }> {
-    // TODO performance
-    const { records: devices } = await this.getAllRecords({
-      name: ELocalDBStoreNames.Device,
-    });
-    devices.forEach((item) => this.refillDeviceInfo({ device: item }));
-    return { devices };
-  }
-
   async getSameDeviceByUUIDEvenIfReset(uuid: string) {
     const { devices } = await this.getAllDevices();
     return devices.find((item) => uuid && item.uuid === uuid);

packages/kit-bg/src/dbs/local/LocalDbBaseContainer.ts

@@ -1,10 +1,14 @@
 import accountUtils from '@onekeyhq/shared/src/utils/accountUtils';
-import { memoizee } from '@onekeyhq/shared/src/utils/cacheUtils';
+import cacheUtils, { memoizee } from '@onekeyhq/shared/src/utils/cacheUtils';
 import timerUtils from '@onekeyhq/shared/src/utils/timerUtils';
 
 import { ELocalDBStoreNames } from './localDBStoreNames';
 
 import type {
+  IDBAccount,
+  IDBDevice,
+  IDBIndexedAccount,
+  IDBWallet,
   ILocalDBAgent,
   ILocalDBGetAllRecordsParams,
   ILocalDBGetAllRecordsResult,
@@ -98,12 +102,25 @@ export abstract class LocalDbBaseContainer implements ILocalDBAgent {
     ].includes(storeName);
   }
 
-  clearStoreCachedData(storeName: ELocalDBStoreNames) {
+  dbAllRecordsCache = new cacheUtils.LRUCache<
+    'allDbAccounts' | 'allDbIndexedAccounts' | 'allDbWallets' | 'allDbDevices',
+    IDBAccount[] | IDBIndexedAccount[] | IDBWallet[] | IDBDevice[]
+  >({
+    max: 10,
+    ttl: timerUtils.getTimeDurationMs({ seconds: 5 }),
+  });
+
+  clearStoreCachedDataIfMatch(storeName: ELocalDBStoreNames) {
     if (this.isCachedStoreName(storeName)) {
-      this.getRecordByIdWithCache.clear();
+      this.clearStoreCachedData();
     }
   }
 
+  clearStoreCachedData() {
+    this.getRecordByIdWithCache.clear();
+    this.dbAllRecordsCache.clear();
+  }
+
   async txGetAllRecords<T extends ELocalDBStoreNames>(
     params: ILocalDBTxGetAllRecordsParams<T>,
   ): Promise<ILocalDBTxGetAllRecordsResult<T>> {
@@ -121,7 +138,7 @@ export abstract class LocalDbBaseContainer implements ILocalDBAgent {
   async txUpdateRecords<T extends ELocalDBStoreNames>(
     params: ILocalDBTxUpdateRecordsParams<T>,
   ): Promise<void> {
-    this.clearStoreCachedData(params.name);
+    this.clearStoreCachedDataIfMatch(params.name);
     const db = await this.readyDb;
     // const a = db.txAddRecords['hello-world-test-error-stack-8889273']['name'];
     return db.txUpdateRecords(params);
@@ -130,23 +147,23 @@ export abstract class LocalDbBaseContainer implements ILocalDBAgent {
   async txAddRecords<T extends ELocalDBStoreNames>(
     params: ILocalDBTxAddRecordsParams<T>,
   ): Promise<ILocalDBTxAddRecordsResult> {
-    this.clearStoreCachedData(params.name);
+    this.clearStoreCachedDataIfMatch(params.name);
     const db = await this.readyDb;
     return db.txAddRecords(params);
   }
 
   async txRemoveRecords<T extends ELocalDBStoreNames>(
     params: ILocalDBTxRemoveRecordsParams<T>,
   ): Promise<void> {
-    this.clearStoreCachedData(params.name);
+    this.clearStoreCachedDataIfMatch(params.name);
     const db = await this.readyDb;
     return db.txRemoveRecords(params);
   }
 
   abstract reset(): Promise<void>;
 
   async clearRecords(params: { name: ELocalDBStoreNames }) {
-    this.clearStoreCachedData(params.name);
+    this.clearStoreCachedDataIfMatch(params.name);
     const db = await this.readyDb;
     return db.clearRecords(params);
   }

packages/kit-bg/src/dbs/local/indexed/IndexedDBAgent.ts

@@ -309,7 +309,7 @@ export class IndexedDBAgent extends LocalDbAgentBase implements ILocalDBAgent {
   ): Promise<ILocalDBTxGetAllRecordsResult<T>> {
     const { tx: paramsTx, name, ids, limit, offset } = params;
     dbPerfMonitor.logLocalDbCall(`txGetAllRecords`, name, [
-      `records: ${ids?.length || ''}`,
+      `ids_count=${ids ? ids?.length?.toString() : 'ALL'}`,
     ]);
     const fn = async (tx: ILocalDBTransaction) => {
       const store = this._getObjectStoreFromTx<T>(tx, name);

packages/kit-bg/src/dbs/local/indexed/LocalDbIndexed.ts

@@ -6,7 +6,7 @@ import { LocalDbIndexedBase } from './LocalDbIndexedBase';
 
 export class LocalDbIndexed extends LocalDbIndexedBase {
   async reset(): Promise<void> {
-    this.clearStoreCachedData(ELocalDBStoreNames.IndexedAccount);
+    this.clearStoreCachedData();
     return this.deleteIndexedDb();
   }
 }

packages/kit-bg/src/dbs/local/realm/LocalDbRealm.ts

@@ -4,7 +4,7 @@ import { LocalDbRealmBase } from './LocalDbRealmBase';
 
 export class LocalDbRealm extends LocalDbRealmBase {
   reset(): Promise<void> {
-    this.clearStoreCachedData(ELocalDBStoreNames.IndexedAccount);
+    this.clearStoreCachedData();
     return this.deleteDb();
   }
 }

packages/kit-bg/src/dbs/local/types.ts

@@ -200,10 +200,12 @@ export type IDBGetWalletsParams = {
   nestedHiddenWallets?: boolean | undefined;
   ignoreEmptySingletonWalletAccounts?: boolean | undefined;
   includingAccounts?: boolean | undefined;
+
+  allIndexedAccounts?: IDBIndexedAccount[] | undefined;
+  allWallets?: IDBWallet[] | undefined;
+  allDevices?: IDBDevice[] | undefined;
 };
-export type IDBGetAllWalletsParams = {
-  refillWalletInfo?: boolean;
-};
+
 // ---------------------------------------------- account
 export type IDBAvatar = string; // stringify(IAvatarInfo)
 // IAvatar;