Skip to content

Commit

Permalink
fix: optimize localDB getAllRecords performance (#6342)
Browse files Browse the repository at this point in the history
  • Loading branch information
@sidmorizon
sidmorizon authored Dec 26, 2024
1 parent 1e050df commit 650eeed
Show file tree
Hide file tree
Showing 31 changed files with 702 additions and 280 deletions.
1 change: 1 addition & 0 deletions apps/desktop/App.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,3 +12,4 @@ import {
initSentry();

export default withSentryHOC(KitProvider);
// export default KitProvider;
1 change: 1 addition & 0 deletions development/spellCheckerSkipWords.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ module.exports = [
'hdk',
'dkey',
'impls',
'ttl',
'Sollet',
'Solflare',
'encryptors',
Expand Down
147 changes: 101 additions & 46 deletions packages/kit-bg/src/dbs/local/LocalDbBase.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,6 @@ import type {
IDBDeviceSettings,
IDBEnsureAccountNameNotDuplicateParams,
IDBExternalAccount,
IDBGetAllWalletsParams,
IDBGetWalletsParams,
IDBIndexedAccount,
IDBRemoveWalletParams,
Expand Down Expand Up @@ -511,30 +510,6 @@ export abstract class LocalDbBase extends LocalDbBaseContainer {
walletSortFn = (a: IDBWallet, b: IDBWallet) =>
(a.walletOrder ?? 0) - (b.walletOrder ?? 0);

async getAllWallets({
refillWalletInfo,
}: IDBGetAllWalletsParams = {}): Promise<{
wallets: IDBWallet[];
}> {
let { records } = await this.getAllRecords({
name: ELocalDBStoreNames.Wallet,
});
if (refillWalletInfo) {
const { devices: allDevices } = await this.getAllDevices();
const refilledWalletsCache: {
[walletId: string]: IDBWallet;
} = {};
records = await Promise.all(
records.map((wallet) =>
this.refillWalletInfo({ wallet, refilledWalletsCache, allDevices }),
),
);
}
return {
wallets: records,
};
}

// eslint-disable-next-line spellcheck/spell-checker
/**
* Get wallets
Expand All @@ -552,8 +527,9 @@ export abstract class LocalDbBase extends LocalDbBaseContainer {
let allIndexedAccounts: IDBIndexedAccount[] | undefined;
if (includingAccounts) {
if (!allIndexedAccounts) {
allIndexedAccounts = (await this.getAllIndexedAccounts())
.indexedAccounts;
allIndexedAccounts =
option?.allIndexedAccounts ||
(await this.getAllIndexedAccounts()).indexedAccounts;
}
}

Expand All @@ -577,8 +553,9 @@ export abstract class LocalDbBase extends LocalDbBaseContainer {
};

// get all wallets for account selector
let { wallets } = await this.getAllWallets();
const { devices: allDevices } = await this.getAllDevices();
let wallets = option?.allWallets || (await this.getAllWallets()).wallets;
const allDevices =
option?.allDevices || (await this.getAllDevices()).devices;
const hiddenWalletsMap: Partial<{
[dbDeviceId: string]: IDBWallet[];
}> = {};
Expand Down Expand Up @@ -2611,7 +2588,8 @@ export abstract class LocalDbBase extends LocalDbBaseContainer {
accounts: IDBAccount[];
}> {
const wallet = await this.getWalletSafe({ walletId });
if (!wallet) {
if (!wallet || !wallet?.accounts?.length) {
// if (!wallet) {
return { accounts: [] };
}
const { accounts } = await this.getAllAccounts({
Expand Down Expand Up @@ -2648,13 +2626,14 @@ export abstract class LocalDbBase extends LocalDbBaseContainer {
const indexedAccount = await this.getIndexedAccount({
id: indexedAccountId,
});
const { accounts } = await this.getAllAccounts();
return accounts
const allDbAccounts = (await this.getAllAccounts()).accounts;
const accounts = allDbAccounts
.filter(
(account) =>
account.indexedAccountId === indexedAccountId && indexedAccountId,
)
.map((account) => this.refillAccountInfo({ account, indexedAccount }));
return { accounts, allDbAccounts };
}

async getAccount({ accountId }: { accountId: string }): Promise<IDBAccount> {
Expand Down Expand Up @@ -2749,18 +2728,101 @@ export abstract class LocalDbBase extends LocalDbBaseContainer {
});
}

async getAllIndexedAccounts() {
async getAllDevices(): Promise<{ devices: IDBDevice[] }> {
const cacheKey = 'allDbDevices';
const allDevicesInCache = this.dbAllRecordsCache.get(
cacheKey,
) as IDBDevice[];
if (allDevicesInCache && allDevicesInCache.length) {
return { devices: allDevicesInCache };
}
const { records: devices } = await this.getAllRecords({
name: ELocalDBStoreNames.Device,
});
devices.forEach((item) => this.refillDeviceInfo({ device: item }));
this.dbAllRecordsCache.set(cacheKey, devices);
return { devices };
}

async getAllWallets(): Promise<{
wallets: IDBWallet[];
}> {
const cacheKey = 'allDbWallets';
const allWalletsInCache = this.dbAllRecordsCache.get(
cacheKey,
) as IDBWallet[];
if (allWalletsInCache && allWalletsInCache.length) {
return { wallets: allWalletsInCache };
}
const { records: wallets } = await this.getAllRecords({
name: ELocalDBStoreNames.Wallet,
});
this.dbAllRecordsCache.set(cacheKey, wallets);
return {
wallets,
};
}

// async getAllWallets({
// refillWalletInfo,
// }: IDBGetAllWalletsParams = {}): Promise<{
// wallets: IDBWallet[];
// }> {
// let { records } = await this.getAllRecords({
// name: ELocalDBStoreNames.Wallet,
// });
// if (refillWalletInfo) {
// const { devices: allDevices } = await this.getAllDevices();
// const refilledWalletsCache: {
// [walletId: string]: IDBWallet;
// } = {};
// records = await Promise.all(
// records.map((wallet) =>
// this.refillWalletInfo({ wallet, refilledWalletsCache, allDevices }),
// ),
// );
// }
// return {
// wallets: records,
// };
// }

async getAllIndexedAccounts(): Promise<{
indexedAccounts: IDBIndexedAccount[];
}> {
const cacheKey = 'allDbIndexedAccounts';
const allIndexedAccountsInCache = this.dbAllRecordsCache.get(
cacheKey,
) as IDBIndexedAccount[];
if (allIndexedAccountsInCache && allIndexedAccountsInCache.length) {
return { indexedAccounts: allIndexedAccountsInCache };
}
const { records: indexedAccounts } = await this.getAllRecords({
name: ELocalDBStoreNames.IndexedAccount,
});
this.dbAllRecordsCache.set(cacheKey, indexedAccounts);
return { indexedAccounts };
}

async getAllAccounts({ ids }: { ids?: string[] } = {}) {
async getAllAccounts({ ids }: { ids?: string[] } = {}): Promise<{
accounts: IDBAccount[];
}> {
const cacheKey = 'allDbAccounts';
if (!ids) {
const allDbAccountsInCache = this.dbAllRecordsCache.get(
cacheKey,
) as IDBAccount[];
if (allDbAccountsInCache && allDbAccountsInCache?.length) {
return { accounts: allDbAccountsInCache };
}
}
const { records: accounts } = await this.getAllRecords({
name: ELocalDBStoreNames.Account,
ids,
});
if (!ids) {
this.dbAllRecordsCache.set(cacheKey, accounts);
}
return { accounts };
}

Expand Down Expand Up @@ -2961,9 +3023,11 @@ export abstract class LocalDbBase extends LocalDbBaseContainer {

if (params.indexedAccountId) {
// TODO low performance
accounts = await this.getAccountsInSameIndexedAccountId({
indexedAccountId: params.indexedAccountId,
});
accounts = (
await this.getAccountsInSameIndexedAccountId({
indexedAccountId: params.indexedAccountId,
})
).accounts;
}
if (params.accountId) {
const account = await this.getAccountSafe({
Expand Down Expand Up @@ -3031,15 +3095,6 @@ export abstract class LocalDbBase extends LocalDbBaseContainer {

// ---------------------------------------------- device

async getAllDevices(): Promise<{ devices: IDBDevice[] }> {
// TODO performance
const { records: devices } = await this.getAllRecords({
name: ELocalDBStoreNames.Device,
});
devices.forEach((item) => this.refillDeviceInfo({ device: item }));
return { devices };
}

async getSameDeviceByUUIDEvenIfReset(uuid: string) {
const { devices } = await this.getAllDevices();
return devices.find((item) => uuid && item.uuid === uuid);
Expand Down
31 changes: 24 additions & 7 deletions packages/kit-bg/src/dbs/local/LocalDbBaseContainer.ts
View file
Original file line number Diff line number Diff line change
@@ -1,10 +1,14 @@
import accountUtils from '@onekeyhq/shared/src/utils/accountUtils';
import { memoizee } from '@onekeyhq/shared/src/utils/cacheUtils';
import cacheUtils, { memoizee } from '@onekeyhq/shared/src/utils/cacheUtils';
import timerUtils from '@onekeyhq/shared/src/utils/timerUtils';

import { ELocalDBStoreNames } from './localDBStoreNames';

import type {
IDBAccount,
IDBDevice,
IDBIndexedAccount,
IDBWallet,
ILocalDBAgent,
ILocalDBGetAllRecordsParams,
ILocalDBGetAllRecordsResult,
Expand Down Expand Up @@ -98,12 +102,25 @@ export abstract class LocalDbBaseContainer implements ILocalDBAgent {
].includes(storeName);
}

clearStoreCachedData(storeName: ELocalDBStoreNames) {
dbAllRecordsCache = new cacheUtils.LRUCache<
'allDbAccounts' | 'allDbIndexedAccounts' | 'allDbWallets' | 'allDbDevices',
IDBAccount[] | IDBIndexedAccount[] | IDBWallet[] | IDBDevice[]
>({
max: 10,
ttl: timerUtils.getTimeDurationMs({ seconds: 5 }),
});

clearStoreCachedDataIfMatch(storeName: ELocalDBStoreNames) {
if (this.isCachedStoreName(storeName)) {
this.getRecordByIdWithCache.clear();
this.clearStoreCachedData();
}
}

clearStoreCachedData() {
this.getRecordByIdWithCache.clear();
this.dbAllRecordsCache.clear();
}

async txGetAllRecords<T extends ELocalDBStoreNames>(
params: ILocalDBTxGetAllRecordsParams<T>,
): Promise<ILocalDBTxGetAllRecordsResult<T>> {
Expand All @@ -121,7 +138,7 @@ export abstract class LocalDbBaseContainer implements ILocalDBAgent {
async txUpdateRecords<T extends ELocalDBStoreNames>(
params: ILocalDBTxUpdateRecordsParams<T>,
): Promise<void> {
this.clearStoreCachedData(params.name);
this.clearStoreCachedDataIfMatch(params.name);
const db = await this.readyDb;
// const a = db.txAddRecords['hello-world-test-error-stack-8889273']['name'];
return db.txUpdateRecords(params);
Expand All @@ -130,23 +147,23 @@ export abstract class LocalDbBaseContainer implements ILocalDBAgent {
async txAddRecords<T extends ELocalDBStoreNames>(
params: ILocalDBTxAddRecordsParams<T>,
): Promise<ILocalDBTxAddRecordsResult> {
this.clearStoreCachedData(params.name);
this.clearStoreCachedDataIfMatch(params.name);
const db = await this.readyDb;
return db.txAddRecords(params);
}

async txRemoveRecords<T extends ELocalDBStoreNames>(
params: ILocalDBTxRemoveRecordsParams<T>,
): Promise<void> {
this.clearStoreCachedData(params.name);
this.clearStoreCachedDataIfMatch(params.name);
const db = await this.readyDb;
return db.txRemoveRecords(params);
}

abstract reset(): Promise<void>;

async clearRecords(params: { name: ELocalDBStoreNames }) {
this.clearStoreCachedData(params.name);
this.clearStoreCachedDataIfMatch(params.name);
const db = await this.readyDb;
return db.clearRecords(params);
}
Expand Down
2 changes: 1 addition & 1 deletion packages/kit-bg/src/dbs/local/indexed/IndexedDBAgent.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -309,7 +309,7 @@ export class IndexedDBAgent extends LocalDbAgentBase implements ILocalDBAgent {
): Promise<ILocalDBTxGetAllRecordsResult<T>> {
const { tx: paramsTx, name, ids, limit, offset } = params;
dbPerfMonitor.logLocalDbCall(`txGetAllRecords`, name, [
`records: ${ids?.length || ''}`,
`ids_count=${ids ? ids?.length?.toString() : 'ALL'}`,
]);
const fn = async (tx: ILocalDBTransaction) => {
const store = this._getObjectStoreFromTx<T>(tx, name);
Expand Down
2 changes: 1 addition & 1 deletion packages/kit-bg/src/dbs/local/indexed/LocalDbIndexed.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ import { LocalDbIndexedBase } from './LocalDbIndexedBase';

export class LocalDbIndexed extends LocalDbIndexedBase {
async reset(): Promise<void> {
this.clearStoreCachedData(ELocalDBStoreNames.IndexedAccount);
this.clearStoreCachedData();
return this.deleteIndexedDb();
}
}
2 changes: 1 addition & 1 deletion packages/kit-bg/src/dbs/local/realm/LocalDbRealm.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ import { LocalDbRealmBase } from './LocalDbRealmBase';

export class LocalDbRealm extends LocalDbRealmBase {
reset(): Promise<void> {
this.clearStoreCachedData(ELocalDBStoreNames.IndexedAccount);
this.clearStoreCachedData();
return this.deleteDb();
}
}
8 changes: 5 additions & 3 deletions packages/kit-bg/src/dbs/local/types.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -200,10 +200,12 @@ export type IDBGetWalletsParams = {
nestedHiddenWallets?: boolean | undefined;
ignoreEmptySingletonWalletAccounts?: boolean | undefined;
includingAccounts?: boolean | undefined;

allIndexedAccounts?: IDBIndexedAccount[] | undefined;
allWallets?: IDBWallet[] | undefined;
allDevices?: IDBDevice[] | undefined;
};
export type IDBGetAllWalletsParams = {
refillWalletInfo?: boolean;
};

// ---------------------------------------------- account
export type IDBAvatar = string; // stringify(IAvatarInfo)
// IAvatar;
Expand Down
Loading

0 comments on commit 650eeed

Please sign in to comment.