fix: optimize localDB getAllRecords performance

OneKeyHQ · Dec 13, 2024 · d4e0de1 · d4e0de1
1 parent 1977cd1
commit d4e0de1
Show file tree

Hide file tree

Showing 31 changed files with 679 additions and 283 deletions.
diff --git a/apps/desktop/App.tsx b/apps/desktop/App.tsx
@@ -12,3 +12,4 @@ import {
 initSentry();
 
 export default withSentryHOC(KitProvider);
+// export default KitProvider;
diff --git a/development/spellCheckerSkipWords.js b/development/spellCheckerSkipWords.js
@@ -20,6 +20,7 @@ module.exports = [
   'hdk',
   'dkey',
   'impls',
+  'ttl',
   'Sollet',
   'Solflare',
   'encryptors',

diff --git a/packages/kit-bg/src/dbs/local/LocalDbBase.ts b/packages/kit-bg/src/dbs/local/LocalDbBase.ts
@@ -56,6 +56,7 @@ import { appLocale } from '@onekeyhq/shared/src/locale/appLocale';
 import accountUtils from '@onekeyhq/shared/src/utils/accountUtils';
 import { checkIsDefined } from '@onekeyhq/shared/src/utils/assertUtils';
 import bufferUtils from '@onekeyhq/shared/src/utils/bufferUtils';
+import cacheUtils from '@onekeyhq/shared/src/utils/cacheUtils';
 import perfUtils, {
   EPerformanceTimerLogNames,
 } from '@onekeyhq/shared/src/utils/debug/perfUtils';
@@ -91,7 +92,6 @@ import type {
   IDBDeviceSettings,
   IDBEnsureAccountNameNotDuplicateParams,
   IDBExternalAccount,
-  IDBGetAllWalletsParams,
   IDBGetWalletsParams,
   IDBIndexedAccount,
   IDBRemoveWalletParams,
@@ -520,31 +520,6 @@ export abstract class LocalDbBase extends LocalDbBaseContainer {
   walletSortFn = (a: IDBWallet, b: IDBWallet) =>
     (a.walletOrder ?? 0) - (b.walletOrder ?? 0);
 
-  async getAllWallets({
-    refillWalletInfo,
-  }: IDBGetAllWalletsParams = {}): Promise<{
-    wallets: IDBWallet[];
-  }> {
-    const db = await this.readyDb;
-    let { records } = await db.getAllRecords({
-      name: ELocalDBStoreNames.Wallet,
-    });
-    if (refillWalletInfo) {
-      const { devices: allDevices } = await this.getAllDevices();
-      const refilledWalletsCache: {
-        [walletId: string]: IDBWallet;
-      } = {};
-      records = await Promise.all(
-        records.map((wallet) =>
-          this.refillWalletInfo({ wallet, refilledWalletsCache, allDevices }),
-        ),
-      );
-    }
-    return {
-      wallets: records,
-    };
-  }
-
   // eslint-disable-next-line spellcheck/spell-checker
   /**
    * Get wallets
@@ -558,13 +533,13 @@ export abstract class LocalDbBase extends LocalDbBaseContainer {
     const ignoreEmptySingletonWalletAccounts =
       option?.ignoreEmptySingletonWalletAccounts;
     const includingAccounts = option?.includingAccounts;
-    const db = await this.readyDb;
 
     let allIndexedAccounts: IDBIndexedAccount[] | undefined;
     if (includingAccounts) {
       if (!allIndexedAccounts) {
-        allIndexedAccounts = (await this.getAllIndexedAccounts())
-          .indexedAccounts;
+        allIndexedAccounts =
+          option?.allIndexedAccounts ||
+          (await this.getAllIndexedAccounts()).indexedAccounts;
       }
     }
 
@@ -588,8 +563,9 @@ export abstract class LocalDbBase extends LocalDbBaseContainer {
     };
 
     // get all wallets for account selector
-    let { wallets } = await this.getAllWallets();
-    const { devices: allDevices } = await this.getAllDevices();
+    let wallets = option?.allWallets || (await this.getAllWallets()).wallets;
+    const allDevices =
+      option?.allDevices || (await this.getAllDevices()).devices;
     const hiddenWalletsMap: Partial<{
       [dbDeviceId: string]: IDBWallet[];
     }> = {};
@@ -2643,9 +2619,9 @@ export abstract class LocalDbBase extends LocalDbBaseContainer {
   }): Promise<{
     accounts: IDBAccount[];
   }> {
-    const db = await this.readyDb;
     const wallet = await this.getWalletSafe({ walletId });
-    if (!wallet) {
+    if (!wallet || !wallet?.accounts?.length) {
+      // if (!wallet) {
       return { accounts: [] };
     }
     const { accounts } = await this.getAllAccounts({
@@ -2682,13 +2658,14 @@ export abstract class LocalDbBase extends LocalDbBaseContainer {
     const indexedAccount = await this.getIndexedAccount({
       id: indexedAccountId,
     });
-    const { accounts } = await this.getAllAccounts();
-    return accounts
+    const allDbAccounts = (await this.getAllAccounts()).accounts;
+    const accounts = allDbAccounts
       .filter(
         (account) =>
           account.indexedAccountId === indexedAccountId && indexedAccountId,
       )
       .map((account) => this.refillAccountInfo({ account, indexedAccount }));
+    return { accounts, allDbAccounts };
   }
 
   async getAccount({ accountId }: { accountId: string }): Promise<IDBAccount> {
@@ -2784,18 +2761,77 @@ export abstract class LocalDbBase extends LocalDbBaseContainer {
     });
   }
 
-  async getAllIndexedAccounts() {
+  async getAllDevices(): Promise<{ devices: IDBDevice[] }> {
+    const cacheKey = 'allDbDevices';
+    const allDevicesInCache = this.dbAllRecordsCache.get(
+      cacheKey,
+    ) as IDBDevice[];
+    if (allDevicesInCache && allDevicesInCache.length) {
+      return { devices: allDevicesInCache };
+    }
+    const { records: devices } = await this.getAllRecords({
+      name: ELocalDBStoreNames.Device,
+    });
+    devices.forEach((item) => this.refillDeviceInfo({ device: item }));
+    this.dbAllRecordsCache.set(cacheKey, devices);
+    return { devices };
+  }
+
+  async getAllWallets(): Promise<{
+    wallets: IDBWallet[];
+  }> {
+    const cacheKey = 'allDbWallets';
+    const allWalletsInCache = this.dbAllRecordsCache.get(
+      cacheKey,
+    ) as IDBWallet[];
+    if (allWalletsInCache && allWalletsInCache.length) {
+      return { wallets: allWalletsInCache };
+    }
+    const { records: wallets } = await this.getAllRecords({
+      name: ELocalDBStoreNames.Wallet,
+    });
+    this.dbAllRecordsCache.set(cacheKey, wallets);
+    return {
+      wallets,
+    };
+  }
+
+  async getAllIndexedAccounts(): Promise<{
+    indexedAccounts: IDBIndexedAccount[];
+  }> {
+    const cacheKey = 'allDbIndexedAccounts';
+    const allIndexedAccountsInCache = this.dbAllRecordsCache.get(
+      cacheKey,
+    ) as IDBIndexedAccount[];
+    if (allIndexedAccountsInCache && allIndexedAccountsInCache.length) {
+      return { indexedAccounts: allIndexedAccountsInCache };
+    }
     const { records: indexedAccounts } = await this.getAllRecords({
       name: ELocalDBStoreNames.IndexedAccount,
     });
+    this.dbAllRecordsCache.set(cacheKey, indexedAccounts);
     return { indexedAccounts };
   }
 
-  async getAllAccounts({ ids }: { ids?: string[] } = {}) {
+  async getAllAccounts({ ids }: { ids?: string[] } = {}): Promise<{
+    accounts: IDBAccount[];
+  }> {
+    const cacheKey = 'allDbAccounts';
+    if (!ids) {
+      const allDbAccountsInCache = this.dbAllRecordsCache.get(
+        cacheKey,
+      ) as IDBAccount[];
+      if (allDbAccountsInCache && allDbAccountsInCache?.length) {
+        return { accounts: allDbAccountsInCache };
+      }
+    }
     const { records: accounts } = await this.getAllRecords({
       name: ELocalDBStoreNames.Account,
       ids,
     });
+    if (!ids) {
+      this.dbAllRecordsCache.set(cacheKey, accounts);
+    }
     return { accounts };
   }
 
@@ -3002,9 +3038,11 @@ export abstract class LocalDbBase extends LocalDbBaseContainer {
 
       if (params.indexedAccountId) {
         // TODO low performance
-        accounts = await this.getAccountsInSameIndexedAccountId({
-          indexedAccountId: params.indexedAccountId,
-        });
+        accounts = (
+          await this.getAccountsInSameIndexedAccountId({
+            indexedAccountId: params.indexedAccountId,
+          })
+        ).accounts;
       }
       if (params.accountId) {
         const account = await this.getAccountSafe({
@@ -3074,15 +3112,6 @@ export abstract class LocalDbBase extends LocalDbBaseContainer {
 
   // ---------------------------------------------- device
 
-  async getAllDevices(): Promise<{ devices: IDBDevice[] }> {
-    // TODO performance
-    const { records: devices } = await this.getAllRecords({
-      name: ELocalDBStoreNames.Device,
-    });
-    devices.forEach((item) => this.refillDeviceInfo({ device: item }));
-    return { devices };
-  }
-
   async getSameDeviceByUUIDEvenIfReset(uuid: string) {
     const { devices } = await this.getAllDevices();
     return devices.find((item) => uuid && item.uuid === uuid);

diff --git a/packages/kit-bg/src/dbs/local/LocalDbBaseContainer.ts b/packages/kit-bg/src/dbs/local/LocalDbBaseContainer.ts
@@ -1,9 +1,13 @@
-import { memoizee } from '@onekeyhq/shared/src/utils/cacheUtils';
+import cacheUtils, { memoizee } from '@onekeyhq/shared/src/utils/cacheUtils';
 import timerUtils from '@onekeyhq/shared/src/utils/timerUtils';
 
 import { ELocalDBStoreNames } from './localDBStoreNames';
 
 import type {
+  IDBAccount,
+  IDBDevice,
+  IDBIndexedAccount,
+  IDBWallet,
   ILocalDBAgent,
   ILocalDBGetAllRecordsParams,
   ILocalDBGetAllRecordsResult,
@@ -88,12 +92,25 @@ export abstract class LocalDbBaseContainer implements ILocalDBAgent {
     ].includes(storeName);
   }
 
-  clearStoreCachedData(storeName: ELocalDBStoreNames) {
+  dbAllRecordsCache = new cacheUtils.LRUCache<
+    'allDbAccounts' | 'allDbIndexedAccounts' | 'allDbWallets' | 'allDbDevices',
+    IDBAccount[] | IDBIndexedAccount[] | IDBWallet[] | IDBDevice[]
+  >({
+    max: 10,
+    ttl: timerUtils.getTimeDurationMs({ seconds: 5 }),
+  });
+
+  clearStoreCachedDataIfMatch(storeName: ELocalDBStoreNames) {
     if (this.isCachedStoreName(storeName)) {
-      this.getRecordByIdWithCache.clear();
+      this.clearStoreCachedData();
     }
   }
 
+  clearStoreCachedData() {
+    this.getRecordByIdWithCache.clear();
+    this.dbAllRecordsCache.clear();
+  }
+
   async txGetAllRecords<T extends ELocalDBStoreNames>(
     params: ILocalDBTxGetAllRecordsParams<T>,
   ): Promise<ILocalDBTxGetAllRecordsResult<T>> {
@@ -111,7 +128,7 @@ export abstract class LocalDbBaseContainer implements ILocalDBAgent {
   async txUpdateRecords<T extends ELocalDBStoreNames>(
     params: ILocalDBTxUpdateRecordsParams<T>,
   ): Promise<void> {
-    this.clearStoreCachedData(params.name);
+    this.clearStoreCachedDataIfMatch(params.name);
     const db = await this.readyDb;
     // const a = db.txAddRecords['hello-world-test-error-stack-8889273']['name'];
     return db.txUpdateRecords(params);
@@ -120,23 +137,23 @@ export abstract class LocalDbBaseContainer implements ILocalDBAgent {
   async txAddRecords<T extends ELocalDBStoreNames>(
     params: ILocalDBTxAddRecordsParams<T>,
   ): Promise<ILocalDBTxAddRecordsResult> {
-    this.clearStoreCachedData(params.name);
+    this.clearStoreCachedDataIfMatch(params.name);
     const db = await this.readyDb;
     return db.txAddRecords(params);
   }
 
   async txRemoveRecords<T extends ELocalDBStoreNames>(
     params: ILocalDBTxRemoveRecordsParams<T>,
   ): Promise<void> {
-    this.clearStoreCachedData(params.name);
+    this.clearStoreCachedDataIfMatch(params.name);
     const db = await this.readyDb;
     return db.txRemoveRecords(params);
   }
 
   abstract reset(): Promise<void>;
 
   async clearRecords(params: { name: ELocalDBStoreNames }) {
-    this.clearStoreCachedData(params.name);
+    this.clearStoreCachedDataIfMatch(params.name);
     const db = await this.readyDb;
     return db.clearRecords(params);
   }

diff --git a/packages/kit-bg/src/dbs/local/indexed/IndexedDBAgent.ts b/packages/kit-bg/src/dbs/local/indexed/IndexedDBAgent.ts
@@ -309,7 +309,7 @@ export class IndexedDBAgent extends LocalDbAgentBase implements ILocalDBAgent {
   ): Promise<ILocalDBTxGetAllRecordsResult<T>> {
     const { tx: paramsTx, name, ids, limit, offset } = params;
     dbPerfMonitor.logLocalDbCall(`txGetAllRecords`, name, [
-      `records: ${ids?.length || ''}`,
+      `ids_count=${ids ? ids?.length?.toString() : 'ALL'}`,
     ]);
     const fn = async (tx: ILocalDBTransaction) => {
       const store = this._getObjectStoreFromTx<T>(tx, name);

diff --git a/packages/kit-bg/src/dbs/local/indexed/LocalDbIndexed.ts b/packages/kit-bg/src/dbs/local/indexed/LocalDbIndexed.ts
@@ -6,7 +6,7 @@ import { LocalDbIndexedBase } from './LocalDbIndexedBase';
 
 export class LocalDbIndexed extends LocalDbIndexedBase {
   async reset(): Promise<void> {
-    this.clearStoreCachedData(ELocalDBStoreNames.IndexedAccount);
+    this.clearStoreCachedData();
     return this.deleteIndexedDb();
   }
 }
diff --git a/packages/kit-bg/src/dbs/local/realm/LocalDbRealm.ts b/packages/kit-bg/src/dbs/local/realm/LocalDbRealm.ts
@@ -4,7 +4,7 @@ import { LocalDbRealmBase } from './LocalDbRealmBase';
 
 export class LocalDbRealm extends LocalDbRealmBase {
   reset(): Promise<void> {
-    this.clearStoreCachedData(ELocalDBStoreNames.IndexedAccount);
+    this.clearStoreCachedData();
     return this.deleteDb();
   }
 }
diff --git a/packages/kit-bg/src/dbs/local/types.ts b/packages/kit-bg/src/dbs/local/types.ts
@@ -200,10 +200,12 @@ export type IDBGetWalletsParams = {
   nestedHiddenWallets?: boolean | undefined;
   ignoreEmptySingletonWalletAccounts?: boolean | undefined;
   includingAccounts?: boolean | undefined;
+
+  allIndexedAccounts?: IDBIndexedAccount[] | undefined;
+  allWallets?: IDBWallet[] | undefined;
+  allDevices?: IDBDevice[] | undefined;
 };
-export type IDBGetAllWalletsParams = {
-  refillWalletInfo?: boolean;
-};
+
 // ---------------------------------------------- account
 export type IDBAvatar = string; // stringify(IAvatarInfo)
 // IAvatar;
Original file line number	Diff line number	Diff line change
Expand Up		@@ -12,3 +12,4 @@ import {
		initSentry();

		export default withSentryHOC(KitProvider);
		// export default KitProvider;