From 268d1c9efb654ba1cfb0b98ce69fd38e8b90a4cd Mon Sep 17 00:00:00 2001 From: Filippo Date: Wed, 13 Nov 2024 12:23:28 +0100 Subject: [PATCH] 256 refactor to enable strict mode (#266) * chore: enable strict mode * fix: matching ipfs metadata string * fix: improve handling of undefined get services * fix: enhanced in memory snapshotting * fix: currency initialization * chore: upgrade indexer version * fix: exceptions handling blockHandlers * fix: snapshotters tests --- src/helpers/stateSnapshot.test.ts | 54 +++------- src/helpers/stateSnapshot.ts | 96 +----------------- src/mappings/handlers/blockHandlers.ts | 132 +++++++++++++++---------- src/mappings/handlers/ethHandlers.ts | 4 +- 4 files changed, 102 insertions(+), 184 deletions(-) diff --git a/src/helpers/stateSnapshot.test.ts b/src/helpers/stateSnapshot.test.ts index 6fe0631..40c34d7 100644 --- a/src/helpers/stateSnapshot.test.ts +++ b/src/helpers/stateSnapshot.test.ts @@ -1,7 +1,7 @@ import { SubstrateBlock } from '@subql/types' import { PoolService } from '../mappings/services/poolService' -import { substrateStateSnapshotter } from './stateSnapshot' -import { Pool, PoolSnapshot } from '../types' +import { BlockInfo, statesSnapshotter } from './stateSnapshot' +import { PoolSnapshot } from '../types' import { getPeriodStart } from './timekeeperService' // eslint-disable-next-line @typescript-eslint/no-explicit-any @@ -25,40 +25,24 @@ describe('Given a populated pool,', () => { set.mockReset() getByFields.mockReset() getByFields.mockReturnValue([pool]) - await substrateStateSnapshotter('periodId', periodId, Pool, PoolSnapshot, block) - expect(store.getByFields).toHaveBeenCalledWith('Pool', [['blockchainId', '=', '0']], expect.anything()) + const blockInfo: BlockInfo = { timestamp, number: blockNumber } + await statesSnapshotter('periodId', periodId, [pool], PoolSnapshot, blockInfo) + expect(store.set).toHaveBeenCalledTimes(2) expect(store.set).toHaveBeenNthCalledWith(1, 'Pool', poolId, expect.anything()) - expect(store.set).toHaveBeenNthCalledWith(2, 'PoolSnapshot', `${poolId}-11246`, expect.anything()) + expect(store.set).toHaveBeenNthCalledWith(2, 'PoolSnapshot', `${poolId}-${blockNumber}`, expect.anything()) }) test('when a snapshot is taken, then the timestamp and blocknumber are added', async () => { set.mockReset() getByFields.mockReset() getByFields.mockReturnValue([pool]) - await substrateStateSnapshotter('periodId', periodId, Pool, PoolSnapshot, block) + const blockInfo: BlockInfo = { timestamp, number: blockNumber } + await statesSnapshotter('periodId', periodId, [pool], PoolSnapshot, blockInfo) expect(store.set).toHaveBeenNthCalledWith( 2, 'PoolSnapshot', - `${poolId}-11246`, - expect.objectContaining({ timestamp: block.timestamp, blockNumber: 11246 }) - ) - }) - - test('when filters are specified, then the correct values are fetched', async () => { - set.mockReset() - getByFields.mockReset() - getByFields.mockReturnValue([pool]) - await substrateStateSnapshotter('periodId', periodId, Pool, PoolSnapshot, block, [ - ['isActive', '=', true], - ]) - expect(store.getByFields).toHaveBeenNthCalledWith( - 1, - 'Pool', - [ - ['blockchainId', '=', '0'], - ['isActive', '=', true], - ], - expect.anything() + `${poolId}-${blockNumber}`, + expect.objectContaining({ timestamp: block.timestamp, blockNumber: blockNumber }) ) }) @@ -66,19 +50,12 @@ describe('Given a populated pool,', () => { set.mockReset() getByFields.mockReset() getByFields.mockReturnValue([pool]) - await substrateStateSnapshotter( - 'periodId', - periodId, - Pool, - PoolSnapshot, - block, - [['type', '=', 'ALL']], - 'poolId' - ) + const blockInfo: BlockInfo = { timestamp, number: blockNumber } + await statesSnapshotter('periodId', periodId, [pool], PoolSnapshot, blockInfo, 'poolId') expect(store.set).toHaveBeenNthCalledWith( 2, 'PoolSnapshot', - `${poolId}-11246`, + `${poolId}-${blockNumber}`, expect.objectContaining({ poolId: poolId }) ) }) @@ -101,13 +78,14 @@ describe('Given a pool with non zero accumulators, ', () => { Object.assign(pool, accumulatedProps) - await substrateStateSnapshotter('periodId', periodId, Pool, PoolSnapshot, block) + const blockInfo: BlockInfo = { timestamp, number: 11246 } + await statesSnapshotter('periodId', periodId, [pool], PoolSnapshot, blockInfo) expect(store.set).toHaveBeenNthCalledWith(1, 'Pool', poolId, expect.objectContaining(zeroedProps)) expect(store.set).toHaveBeenNthCalledWith( 2, 'PoolSnapshot', - `${poolId}-11246`, + `${poolId}-${blockNumber}`, expect.objectContaining(zeroedProps) ) }) diff --git a/src/helpers/stateSnapshot.ts b/src/helpers/stateSnapshot.ts index 9c41e7e..4ac095e 100644 --- a/src/helpers/stateSnapshot.ts +++ b/src/helpers/stateSnapshot.ts @@ -1,7 +1,4 @@ -import { paginatedGetter } from './paginatedGetter' import type { Entity, FieldsExpression, FunctionPropertyNames, GetOptions } from '@subql/types-core' -import { EthereumBlock } from '@subql/types-ethereum' -import { SubstrateBlock } from '@subql/types' /** * Creates a snapshot of a generic stateModel to a snapshotModel. * A snapshotModel has the same fields as the originating stateModel, however a timestamp and a blockNumber are added. @@ -16,72 +13,6 @@ import { SubstrateBlock } from '@subql/types' * @param resetPeriodStates - (optional) reset properties ending in ByPeriod to 0 after snapshot (defaults to true). * @returns A promise resolving when all state manipulations in the DB is completed */ -async function stateSnapshotter>( - relationshipField: StringForeignKeys, - relationshipId: string, - stateModel: EntityClass, - snapshotModel: EntityClass, - block: { number: number; timestamp: Date }, - filters?: FieldsExpression>[], - fkReferenceField?: StringForeignKeys, - resetPeriodStates = true, - blockchainId = '0' -): Promise { - const entitySaves: Promise[] = [] - logger.info(`Performing snapshots of ${stateModel.prototype._name} for blockchainId ${blockchainId}`) - const filter = [['blockchainId', '=', blockchainId]] as FieldsExpression>[] - if (filters) filter.push(...filters) - const stateEntities = await paginatedGetter(stateModel, filter) - if (stateEntities.length === 0) logger.info(`No ${stateModel.prototype._name} to snapshot!`) - for (const stateEntity of stateEntities) { - const blockNumber = block.number - const snapshot: SnapshottedEntity = { - ...stateEntity, - id: `${stateEntity.id}-${blockNumber}`, - timestamp: block.timestamp, - blockNumber: blockNumber, - [relationshipField]: relationshipId, - } - logger.info(`Snapshotting ${stateModel.prototype._name}: ${stateEntity.id}`) - const snapshotEntity = snapshotModel.create(snapshot as U) - if (fkReferenceField) snapshotEntity[fkReferenceField] = stateEntity.id as U[StringForeignKeys] - const propNames = Object.getOwnPropertyNames(stateEntity) - const propNamesToReset = propNames.filter((propName) => propName.endsWith('ByPeriod')) as ResettableKeys[] - if (resetPeriodStates) { - for (const propName of propNamesToReset) { - logger.debug(`resetting ${stateEntity._name?.toLowerCase()}.${propName} to 0`) - stateEntity[propName] = BigInt(0) as T[ResettableKeys] - } - entitySaves.push(stateEntity.save()) - } - entitySaves.push(snapshotEntity.save()) - } - return Promise.all(entitySaves) -} -export function evmStateSnapshotter>( - relationshipField: StringForeignKeys, - relationshipId: string, - stateModel: EntityClass, - snapshotModel: EntityClass, - block: EthereumBlock, - filters?: FieldsExpression>[], - fkReferenceName?: StringForeignKeys, - resetPeriodStates = true -): Promise { - const formattedBlock = { number: block.number, timestamp: new Date(Number(block.timestamp) * 1000) } - return stateSnapshotter( - relationshipField, - relationshipId, - stateModel, - snapshotModel, - formattedBlock, - filters, - fkReferenceName, - resetPeriodStates, - '1' - ) -} - export async function statesSnapshotter>( relationshipField: StringForeignKeys, relationshipId: string, @@ -116,35 +47,12 @@ export async function statesSnapshotter>( - relationshipField: StringForeignKeys, - relationshipId: string, - stateModel: EntityClass, - snapshotModel: EntityClass, - block: SubstrateBlock, - filters?: FieldsExpression>[], - fkReferenceName?: StringForeignKeys, - resetPeriodStates = true -): Promise { - if (!block.timestamp) throw new Error('Missing block timestamp') - const formattedBlock = { number: block.block.header.number.toNumber(), timestamp: block.timestamp } - return stateSnapshotter( - relationshipField, - relationshipId, - stateModel, - snapshotModel, - formattedBlock, - filters, - fkReferenceName, - resetPeriodStates, - '0' - ) -} - type ResettableKeyFormat = `${string}ByPeriod` type ForeignKeyFormat = `${string}Id` type ResettableKeys = { [K in keyof T]: K extends ResettableKeyFormat ? K : never }[keyof T] diff --git a/src/mappings/handlers/blockHandlers.ts b/src/mappings/handlers/blockHandlers.ts index db9eecd..0b05f2f 100644 --- a/src/mappings/handlers/blockHandlers.ts +++ b/src/mappings/handlers/blockHandlers.ts @@ -52,9 +52,18 @@ async function _handleBlock(block: SubstrateBlock): Promise { const pools = await PoolService.getCfgActivePools() for (const pool of pools) { logger.info(` ## Updating pool ${pool.id} states...`) - if (!pool.currentEpoch) throw new Error('Pool currentEpoch not set') + + if (!pool.currentEpoch) { + logger.error(`Pool currentEpoch not set for ${pool.id}, skipping...`) + continue + } + const currentEpoch = await EpochService.getById(pool.id, pool.currentEpoch) - if (!currentEpoch) throw new Error(`Current epoch ${pool.currentEpoch} for pool ${pool.id} not found`) + if (!currentEpoch) { + logger.error(`Current epoch ${pool.currentEpoch} for pool ${pool.id} not found, skipping pool`) + continue + } + await pool.updateState() await pool.resetDebtOverdue() @@ -63,8 +72,14 @@ async function _handleBlock(block: SubstrateBlock): Promise { const trancheData = await pool.getTranches() const trancheTokenPrices = await pool.getTrancheTokenPrices() for (const tranche of tranches) { - if (typeof tranche.index !== 'number') throw new Error('Tranche index not set') - if (!trancheTokenPrices) break + if (typeof tranche.index !== 'number') { + logger.error('Tranche index not set, skipping tranche') + continue + } + if (!trancheTokenPrices) { + logger.error('trancheTokenPrices not available, skipping tranche updates') + break + } await tranche.updatePrice(trancheTokenPrices[tranche.index].toBigInt(), block.block.header.number.toNumber()) await tranche.updateSupply() await tranche.updateDebt(trancheData[tranche.trancheId].debt) @@ -84,7 +99,12 @@ async function _handleBlock(block: SubstrateBlock): Promise { limit: 100, })) as TrancheBalanceService[] for (const trancheBalance of trancheBalances) { - if (!tranche.tokenPrice) throw new Error('Tranche token price not set') + if (typeof tranche.tokenPrice !== 'bigint') { + console.warn( + `tokenPrice not set, unable to update unrealizedProfit for trancheBalance ${trancheBalance.id}` + ) + continue + } const unrealizedProfit = await InvestorPositionService.computeUnrealizedProfitAtPrice( trancheBalance.accountId, tranche.id, @@ -95,50 +115,61 @@ async function _handleBlock(block: SubstrateBlock): Promise { } } // Asset operations - const activeLoanData = await pool.getPortfolio() + const activeAssetData = await pool.getPortfolio() pool.resetOffchainCashValue() pool.resetUnrealizedProfit() - for (const loanId in activeLoanData) { - const asset = await AssetService.getById(pool.id, loanId) + for (const assetId in activeAssetData) { + const asset = await AssetService.getById(pool.id, assetId) if (!asset) continue - if (!asset.currentPrice) throw new Error('Asset current price not set') - if (!asset.notional) throw new Error('Asset notional not set') await asset.loadSnapshot(lastPeriodStart) - await asset.updateActiveAssetData(activeLoanData[loanId]) - await asset.updateUnrealizedProfit( - await AssetPositionService.computeUnrealizedProfitAtPrice(asset.id, asset.currentPrice), - await AssetPositionService.computeUnrealizedProfitAtPrice(asset.id, asset.notional) - ) + await asset.updateActiveAssetData(activeAssetData[assetId]) + if (asset.notional && asset.currentPrice) { + await asset.updateUnrealizedProfit( + await AssetPositionService.computeUnrealizedProfitAtPrice(asset.id, asset.currentPrice), + await AssetPositionService.computeUnrealizedProfitAtPrice(asset.id, asset.notional) + ) + } else { + console.warn(`Missing current price or notional, unable to update unrealized profit for asset ${assetId}`) + } await asset.save() assetsToSnapshot.push(asset) - if (typeof asset.interestAccruedByPeriod !== 'bigint') - throw new Error('Asset interest accrued by period not set') - await pool.increaseInterestAccrued(asset.interestAccruedByPeriod) - + if (typeof asset.interestAccruedByPeriod === 'bigint') { + await pool.increaseInterestAccrued(asset.interestAccruedByPeriod) + } else { + logger.warn(`interestAccruedByPeriod not set, unable to compute accrued interest for asset ${assetId}`) + } if (asset.isNonCash()) { - if (typeof asset.unrealizedProfitAtMarketPrice !== 'bigint') - throw new Error('Asset unrealized profit at market price not set') - if (typeof asset.unrealizedProfitAtNotional !== 'bigint') - throw new Error('Asset unrealized profit at notional not set') - if (typeof asset.unrealizedProfitByPeriod !== 'bigint') - throw new Error('Asset unrealized profit by period not set') - pool.increaseUnrealizedProfit( - asset.unrealizedProfitAtMarketPrice, - asset.unrealizedProfitAtNotional, - asset.unrealizedProfitByPeriod - ) + if ( + typeof asset.unrealizedProfitAtMarketPrice === 'bigint' && + typeof asset.unrealizedProfitAtNotional === 'bigint' && + typeof asset.unrealizedProfitByPeriod === 'bigint' + ) { + pool.increaseUnrealizedProfit( + asset.unrealizedProfitAtMarketPrice, + asset.unrealizedProfitAtNotional, + asset.unrealizedProfitByPeriod + ) + } else { + logger.warn(`Missing unrealized profit figures, unable to increase unrealized profit for asset ${assetId}`) + } } if (asset.isBeyondMaturity(block.timestamp)) { - if (typeof asset.outstandingDebt !== 'bigint') throw new Error('Asset outstanding debt not set') - pool.increaseDebtOverdue(asset.outstandingDebt) + if (typeof asset.outstandingDebt === 'bigint') { + pool.increaseDebtOverdue(asset.outstandingDebt) + } else { + logger.warn(`Unable to increase debt overdue, missing outstandingDebt for ${assetId}`) + } } if (asset.isOffchainCash()) { - if (typeof asset.presentValue !== 'bigint') throw new Error('Asset present value not set') - pool.increaseOffchainCashValue(asset.presentValue) + if (typeof asset.presentValue === 'bigint') { + pool.increaseOffchainCashValue(asset.presentValue) + } else { + logger.warn(`Asset present value not set, unable to increase offchain cash value for ${assetId}`) + } } } - await pool.updateNumberOfActiveAssets(BigInt(Object.keys(activeLoanData).length)) + await pool.updateNumberOfActiveAssets(BigInt(Object.keys(activeAssetData).length)) // NAV update requires updated offchain cash value await pool.updateNAV() @@ -156,20 +187,21 @@ async function _handleBlock(block: SubstrateBlock): Promise { await poolFee.save() poolFeesToSnapshot.push(poolFee) - if (typeof poolFee.sumAccruedAmountByPeriod !== 'bigint') - throw new Error('Pool fee sum accrued amount by period not set') - await pool.increaseAccruedFees(poolFee.sumAccruedAmountByPeriod) - - const poolFeeTransaction = PoolFeeTransactionService.accrue({ - poolId: pool.id, - feeId, - blockNumber, - amount: poolFee.sumAccruedAmountByPeriod, - epochId: currentEpoch.id, - hash: block.hash.toHex(), - timestamp: block.timestamp, - }) - await poolFeeTransaction.save() + if (typeof poolFee.sumAccruedAmountByPeriod === 'bigint') { + await pool.increaseAccruedFees(poolFee.sumAccruedAmountByPeriod) + const poolFeeTransaction = PoolFeeTransactionService.accrue({ + poolId: pool.id, + feeId, + blockNumber, + amount: poolFee.sumAccruedAmountByPeriod, + epochId: currentEpoch.id, + hash: block.hash.toHex(), + timestamp: block.timestamp, + }) + await poolFeeTransaction.save() + } else { + logger.warn(`sumAccruedAmountByPeriod not set. unable to increase accrued fees for ${poolFee.id}`) + } } const sumPoolFeesPendingAmount = await PoolFeeService.computeSumPendingFees(pool.id) await pool.updateSumPoolFeesPendingAmount(sumPoolFeesPendingAmount) @@ -180,7 +212,7 @@ async function _handleBlock(block: SubstrateBlock): Promise { logger.info('## Performing snapshots...') const blockInfo = { number: block.block.header.number.toNumber(), timestamp: block.timestamp } - await statesSnapshotter('periodId', period.id, pools, PoolSnapshot, blockInfo, 'poolId') + await statesSnapshotter('periodId', period.id, poolsToSnapshot, PoolSnapshot, blockInfo, 'poolId') await statesSnapshotter('periodId', period.id, tranchesToSnapshot, TrancheSnapshot, blockInfo, 'trancheId') await statesSnapshotter('periodId', period.id, assetsToSnapshot, AssetSnapshot, blockInfo, 'assetId') await statesSnapshotter('periodId', period.id, poolFeesToSnapshot, PoolFeeSnapshot, blockInfo, 'poolFeeId') diff --git a/src/mappings/handlers/ethHandlers.ts b/src/mappings/handlers/ethHandlers.ts index a67629a..c6b6de9 100644 --- a/src/mappings/handlers/ethHandlers.ts +++ b/src/mappings/handlers/ethHandlers.ts @@ -46,7 +46,7 @@ async function _handleEthBlock(block: EthereumBlock): Promise { PoolService['id'], { pool: PoolService - tinlakePool: typeof tinlakePools[0] + tinlakePool: (typeof tinlakePools)[0] latestNavFeed?: ContractArray latestReserve?: ContractArray } @@ -152,7 +152,7 @@ async function _handleEthBlock(block: EthereumBlock): Promise { // Take snapshots const blockInfo: BlockInfo = { timestamp: date, number: block.number } - const poolsToSnapshot: PoolService[] = Object.values(processedPools).map(e => e.pool) + const poolsToSnapshot: PoolService[] = Object.values(processedPools).map((e) => e.pool) await statesSnapshotter('periodId', snapshotPeriod.id, poolsToSnapshot, PoolSnapshot, blockInfo, 'poolId') //Update tracking of period and continue