From 1a903fccc6f4c1e2ee4d518c4b6f903e340843b0 Mon Sep 17 00:00:00 2001 From: nikugogoi Date: Mon, 8 Aug 2022 15:58:13 +0530 Subject: [PATCH] CLI to compare eden-watcher entities and fix mapping code (#149) * Make sumStaked variable local in eden network mapping * Implement compare CLI to fetch and query by ids * Set filterLogs to true for eden-watcher * Use varchar for bigint array type in eden-watcher * Store subgraph entities by id in IPLD state * Store bigint vales as string in IPLD state * Update eden watcher hook to store single Block entity in IPLD checkpoint * Fix entity enum type property * Fix parsing big numbers in event params * Fix event bigint params parsing in all watchers * Set default limit to query result and process block after events --- .../src/templates/indexer-template.handlebars | 21 ++++--- packages/eden-watcher/environments/local.toml | 8 +-- packages/eden-watcher/src/entity/Network.ts | 3 +- .../src/entity/ProducerSetChange.ts | 4 +- .../eden-watcher/src/gql/queries/account.gql | 4 +- .../eden-watcher/src/gql/queries/block.gql | 4 +- .../eden-watcher/src/gql/queries/claim.gql | 4 +- .../src/gql/queries/distribution.gql | 4 +- .../src/gql/queries/distributor.gql | 4 +- .../eden-watcher/src/gql/queries/epoch.gql | 4 +- .../eden-watcher/src/gql/queries/network.gql | 4 +- .../eden-watcher/src/gql/queries/producer.gql | 4 +- .../src/gql/queries/producerEpoch.gql | 4 +- .../queries/producerRewardCollectorChange.gql | 4 +- .../src/gql/queries/producerSet.gql | 4 +- .../src/gql/queries/producerSetChange.gql | 4 +- .../src/gql/queries/rewardSchedule.gql | 4 +- .../src/gql/queries/rewardScheduleEntry.gql | 4 +- .../eden-watcher/src/gql/queries/slash.gql | 4 +- .../eden-watcher/src/gql/queries/slot.gql | 4 +- .../src/gql/queries/slotClaim.gql | 4 +- .../eden-watcher/src/gql/queries/staker.gql | 4 +- packages/eden-watcher/src/hooks.ts | 58 +++++++++++++++++- packages/eden-watcher/src/indexer.ts | 23 ++++--- packages/erc20-watcher/src/indexer.ts | 17 ++--- packages/erc721-watcher/src/indexer.ts | 41 +++++++------ packages/graph-node/README.md | 43 +++++++++++++ .../environments/compare-cli-config.toml | 1 + packages/graph-node/package.json | 2 + packages/graph-node/src/cli/compare/client.ts | 42 +++++++++++++ .../src/cli/compare/compare-blocks.ts | 55 +++++++++++++++-- .../src/cli/compare/compare-entity.ts | 3 +- packages/graph-node/src/cli/compare/utils.ts | 3 +- packages/graph-node/src/database.ts | 8 ++- packages/graph-node/src/loader.ts | 9 ++- packages/graph-node/src/watcher.ts | 2 +- .../eden/EdenNetwork/EdenNetwork.wasm | Bin 155010 -> 155014 bytes .../EdenNetworkGovernance.wasm | Bin 174794 -> 174491 bytes packages/graph-test-watcher/src/indexer.ts | 19 +++--- packages/lighthouse-watcher/src/indexer.ts | 3 +- packages/mobymask-watcher/src/indexer.ts | 15 ++--- packages/solidity-mapper/src/storage.test.ts | 4 +- packages/uni-info-watcher/src/indexer.ts | 11 ++-- packages/uni-watcher/src/indexer.ts | 13 ++-- packages/util/src/common.ts | 4 ++ packages/util/src/types.ts | 1 + yarn.lock | 10 +++ 47 files changed, 363 insertions(+), 132 deletions(-) diff --git a/packages/codegen/src/templates/indexer-template.handlebars b/packages/codegen/src/templates/indexer-template.handlebars index 434129f24..6b2bf38ff 100644 --- a/packages/codegen/src/templates/indexer-template.handlebars +++ b/packages/codegen/src/templates/indexer-template.handlebars @@ -52,6 +52,7 @@ import { {{subgraphEntity.className}} } from './entity/{{subgraphEntity.classNam {{/each}} const log = debug('vulcanize:indexer'); +const JSONbigNative = JSONbig({ useNativeBigInt: true }); {{#each contracts as | contract |}} const KIND_{{capitalize contract.contractName}} = '{{contract.contractKind}}'; @@ -177,8 +178,8 @@ export class Indexer implements IPLDIndexerInterface { getResultEvent (event: Event): ResultEvent { const block = event.block; - const eventFields = JSONbig.parse(event.eventInfo); - const { tx, eventSignature } = JSON.parse(event.extraInfo); + const eventFields = JSONbigNative.parse(event.eventInfo); + const { tx, eventSignature } = JSONbigNative.parse(event.extraInfo); return { block: { @@ -288,7 +289,7 @@ export class Indexer implements IPLDIndexerInterface { {{/if}} await this._db.{{query.saveQueryName}}({ blockHash, blockNumber, contractAddress - {{~#each query.params}}, {{this.name~}} {{/each}}, value: result.value, proof: JSONbig.stringify(result.proof) }); + {{~#each query.params}}, {{this.name~}} {{/each}}, value: result.value, proof: JSONbigNative.stringify(result.proof) }); {{#if query.stateVariableType}} {{#if (compare query.stateVariableType 'Mapping')}} @@ -441,13 +442,15 @@ export class Indexer implements IPLDIndexerInterface { async processBlock (blockHash: string, blockNumber: number): Promise { // Call a function to create initial state for contracts. await this._baseIndexer.createInit(this, blockHash, blockNumber); - {{#if (subgraphPath)}} + } + {{#if (subgraphPath)}} + async processBlockAfterEvents (blockHash: string): Promise { // Call subgraph handler for block. await this._graphWatcher.handleBlock(blockHash); - {{/if}} } + {{/if}} parseEventNameAndArgs (kind: string, logObj: any): any { const { topics, data } = logObj; @@ -770,10 +773,10 @@ export class Indexer implements IPLDIndexerInterface { txHash, contract, eventName, - eventInfo: JSONbig.stringify(eventInfo), - extraInfo: JSONbig.stringify(extraInfo), - proof: JSONbig.stringify({ - data: JSONbig.stringify({ + eventInfo: JSONbigNative.stringify(eventInfo), + extraInfo: JSONbigNative.stringify(extraInfo), + proof: JSONbigNative.stringify({ + data: JSONbigNative.stringify({ blockHash, receiptCID, log: { diff --git a/packages/eden-watcher/environments/local.toml b/packages/eden-watcher/environments/local.toml index 21d556015..64f6af105 100644 --- a/packages/eden-watcher/environments/local.toml +++ b/packages/eden-watcher/environments/local.toml @@ -10,13 +10,13 @@ checkpointInterval = 2000 # IPFS API address (can be taken from the output on running the IPFS daemon). - ipfsApiAddr = "/ip4/127.0.0.1/tcp/5001" + # ipfsApiAddr = "/ip4/127.0.0.1/tcp/5001" subgraphPath = "../graph-node/test/subgraph/eden" wasmRestartBlocksInterval = 20 # Boolean to filter logs by contract. - filterLogs = false + filterLogs = true # Max block range for which to return events in eventsInRange GQL query. # Use -1 for skipping check on block range. @@ -38,8 +38,8 @@ [upstream] [upstream.ethServer] - gqlApiEndpoint = "http://127.0.0.1:8082/graphql" - rpcProviderEndpoint = "http://127.0.0.1:8081" + gqlApiEndpoint = "http://127.0.0.1:8083/graphql" + rpcProviderEndpoint = "http://127.0.0.1:8082" blockDelayInMilliSecs = 2000 [upstream.cache] diff --git a/packages/eden-watcher/src/entity/Network.ts b/packages/eden-watcher/src/entity/Network.ts index 0792dec4b..4de8f07e1 100644 --- a/packages/eden-watcher/src/entity/Network.ts +++ b/packages/eden-watcher/src/entity/Network.ts @@ -35,6 +35,7 @@ export class Network { @Column('numeric', { transformer: bigintTransformer }) totalStaked!: bigint; - @Column('numeric', { transformer: bigintArrayTransformer, array: true }) + // https://github.com/brianc/node-postgres/issues/1943#issuecomment-520500053 + @Column('varchar', { transformer: bigintArrayTransformer, array: true }) stakedPercentiles!: bigint[]; } diff --git a/packages/eden-watcher/src/entity/ProducerSetChange.ts b/packages/eden-watcher/src/entity/ProducerSetChange.ts index 1d024fded..8f0010c76 100644 --- a/packages/eden-watcher/src/entity/ProducerSetChange.ts +++ b/packages/eden-watcher/src/entity/ProducerSetChange.ts @@ -6,8 +6,8 @@ import { Entity, PrimaryColumn, Column } from 'typeorm'; import { bigintTransformer } from '@vulcanize/util'; enum ProducerSetChangeType { - Added, - Removed + Added = 'Added', + Removed = 'Removed' } @Entity() diff --git a/packages/eden-watcher/src/gql/queries/account.gql b/packages/eden-watcher/src/gql/queries/account.gql index 28eb1460a..c07f1a39d 100644 --- a/packages/eden-watcher/src/gql/queries/account.gql +++ b/packages/eden-watcher/src/gql/queries/account.gql @@ -1,5 +1,5 @@ -query account($id: String!, $blockHash: Bytes!){ - account(id: $id, block: { hash: $blockHash }){ +query account($id: String!, $block: Block_height){ + account(id: $id, block: $block){ id totalClaimed totalSlashed diff --git a/packages/eden-watcher/src/gql/queries/block.gql b/packages/eden-watcher/src/gql/queries/block.gql index 0c41441f6..ab1c9ae45 100644 --- a/packages/eden-watcher/src/gql/queries/block.gql +++ b/packages/eden-watcher/src/gql/queries/block.gql @@ -1,5 +1,5 @@ -query block($id: String!, $blockHash: Bytes!){ - block(id: $id, block: { hash: $blockHash }){ +query block($id: String!, $block: Block_height){ + block(id: $id, block: $block){ id fromActiveProducer hash diff --git a/packages/eden-watcher/src/gql/queries/claim.gql b/packages/eden-watcher/src/gql/queries/claim.gql index 6f7435fbf..4b79c418c 100644 --- a/packages/eden-watcher/src/gql/queries/claim.gql +++ b/packages/eden-watcher/src/gql/queries/claim.gql @@ -1,5 +1,5 @@ -query claim($id: String!, $blockHash: Bytes!){ - claim(id: $id, block: { hash: $blockHash }){ +query claim($id: String!, $block: Block_height){ + claim(id: $id, block: $block){ id timestamp index diff --git a/packages/eden-watcher/src/gql/queries/distribution.gql b/packages/eden-watcher/src/gql/queries/distribution.gql index 877677cb6..b838cca2c 100644 --- a/packages/eden-watcher/src/gql/queries/distribution.gql +++ b/packages/eden-watcher/src/gql/queries/distribution.gql @@ -1,5 +1,5 @@ -query distribution($id: String!, $blockHash: Bytes!){ - distribution(id: $id, block: { hash: $blockHash }){ +query distribution($id: String!, $block: Block_height){ + distribution(id: $id, block: $block){ id distributor{ id diff --git a/packages/eden-watcher/src/gql/queries/distributor.gql b/packages/eden-watcher/src/gql/queries/distributor.gql index 6df2d1d85..575703f4b 100644 --- a/packages/eden-watcher/src/gql/queries/distributor.gql +++ b/packages/eden-watcher/src/gql/queries/distributor.gql @@ -1,5 +1,5 @@ -query distributor($id: String!, $blockHash: Bytes!){ - distributor(id: $id, block: { hash: $blockHash }){ +query distributor($id: String!, $block: Block_height){ + distributor(id: $id, block: $block){ id currentDistribution{ id diff --git a/packages/eden-watcher/src/gql/queries/epoch.gql b/packages/eden-watcher/src/gql/queries/epoch.gql index 538996673..5d8789652 100644 --- a/packages/eden-watcher/src/gql/queries/epoch.gql +++ b/packages/eden-watcher/src/gql/queries/epoch.gql @@ -1,5 +1,5 @@ -query epoch($id: String!, $blockHash: Bytes!){ - epoch(id: $id, block: { hash: $blockHash }){ +query epoch($id: String!, $block: Block_height){ + epoch(id: $id, block: $block){ id finalized epochNumber diff --git a/packages/eden-watcher/src/gql/queries/network.gql b/packages/eden-watcher/src/gql/queries/network.gql index 4992bf356..e5daf160b 100644 --- a/packages/eden-watcher/src/gql/queries/network.gql +++ b/packages/eden-watcher/src/gql/queries/network.gql @@ -1,5 +1,5 @@ -query network($id: String!, $blockHash: Bytes!){ - network(id: $id, block: { hash: $blockHash }){ +query network($id: String!, $block: Block_height){ + network(id: $id, block: $block){ id slot0{ id diff --git a/packages/eden-watcher/src/gql/queries/producer.gql b/packages/eden-watcher/src/gql/queries/producer.gql index 8332995e1..47fbe7ee3 100644 --- a/packages/eden-watcher/src/gql/queries/producer.gql +++ b/packages/eden-watcher/src/gql/queries/producer.gql @@ -1,5 +1,5 @@ -query producer($id: String!, $blockHash: Bytes!){ - producer(id: $id, block: { hash: $blockHash }){ +query producer($id: String!, $block: Block_height){ + producer(id: $id, block: $block){ id active rewardCollector diff --git a/packages/eden-watcher/src/gql/queries/producerEpoch.gql b/packages/eden-watcher/src/gql/queries/producerEpoch.gql index 836f045e8..9cd267ee4 100644 --- a/packages/eden-watcher/src/gql/queries/producerEpoch.gql +++ b/packages/eden-watcher/src/gql/queries/producerEpoch.gql @@ -1,5 +1,5 @@ -query producerEpoch($id: String!, $blockHash: Bytes!){ - producerEpoch(id: $id, block: { hash: $blockHash }){ +query producerEpoch($id: String!, $block: Block_height){ + producerEpoch(id: $id, block: $block){ id address epoch{ diff --git a/packages/eden-watcher/src/gql/queries/producerRewardCollectorChange.gql b/packages/eden-watcher/src/gql/queries/producerRewardCollectorChange.gql index cf3e96d94..249c419c6 100644 --- a/packages/eden-watcher/src/gql/queries/producerRewardCollectorChange.gql +++ b/packages/eden-watcher/src/gql/queries/producerRewardCollectorChange.gql @@ -1,5 +1,5 @@ -query producerRewardCollectorChange($id: String!, $blockHash: Bytes!){ - producerRewardCollectorChange(id: $id, block: { hash: $blockHash }){ +query producerRewardCollectorChange($id: String!, $block: Block_height){ + producerRewardCollectorChange(id: $id, block: $block){ id blockNumber producer diff --git a/packages/eden-watcher/src/gql/queries/producerSet.gql b/packages/eden-watcher/src/gql/queries/producerSet.gql index f2593103f..33ab68948 100644 --- a/packages/eden-watcher/src/gql/queries/producerSet.gql +++ b/packages/eden-watcher/src/gql/queries/producerSet.gql @@ -1,5 +1,5 @@ -query producerSet($id: String!, $blockHash: Bytes!){ - producerSet(id: $id, block: { hash: $blockHash }){ +query producerSet($id: String!, $block: Block_height){ + producerSet(id: $id, block: $block){ id producers{ id diff --git a/packages/eden-watcher/src/gql/queries/producerSetChange.gql b/packages/eden-watcher/src/gql/queries/producerSetChange.gql index 8d89193d4..2e79f6385 100644 --- a/packages/eden-watcher/src/gql/queries/producerSetChange.gql +++ b/packages/eden-watcher/src/gql/queries/producerSetChange.gql @@ -1,5 +1,5 @@ -query producerSetChange($id: String!, $blockHash: Bytes!){ - producerSetChange(id: $id, block: { hash: $blockHash }){ +query producerSetChange($id: String!, $block: Block_height){ + producerSetChange(id: $id, block: $block){ id blockNumber producer diff --git a/packages/eden-watcher/src/gql/queries/rewardSchedule.gql b/packages/eden-watcher/src/gql/queries/rewardSchedule.gql index 226c29dcf..bf4de64a4 100644 --- a/packages/eden-watcher/src/gql/queries/rewardSchedule.gql +++ b/packages/eden-watcher/src/gql/queries/rewardSchedule.gql @@ -1,5 +1,5 @@ -query rewardSchedule($id: String!, $blockHash: Bytes!){ - rewardSchedule(id: $id, block: { hash: $blockHash }){ +query rewardSchedule($id: String!, $block: Block_height){ + rewardSchedule(id: $id, block: $block){ id rewardScheduleEntries{ id diff --git a/packages/eden-watcher/src/gql/queries/rewardScheduleEntry.gql b/packages/eden-watcher/src/gql/queries/rewardScheduleEntry.gql index 666e1b297..ecf4aee20 100644 --- a/packages/eden-watcher/src/gql/queries/rewardScheduleEntry.gql +++ b/packages/eden-watcher/src/gql/queries/rewardScheduleEntry.gql @@ -1,5 +1,5 @@ -query rewardScheduleEntry($id: String!, $blockHash: Bytes!){ - rewardScheduleEntry(id: $id, block: { hash: $blockHash }){ +query rewardScheduleEntry($id: String!, $block: Block_height){ + rewardScheduleEntry(id: $id, block: $block){ id startTime epochDuration diff --git a/packages/eden-watcher/src/gql/queries/slash.gql b/packages/eden-watcher/src/gql/queries/slash.gql index 62b2840b2..2e9692a18 100644 --- a/packages/eden-watcher/src/gql/queries/slash.gql +++ b/packages/eden-watcher/src/gql/queries/slash.gql @@ -1,5 +1,5 @@ -query slash($id: String!, $blockHash: Bytes!){ - slash(id: $id, block: { hash: $blockHash }){ +query slash($id: String!, $block: Block_height){ + slash(id: $id, block: $block){ id timestamp account{ diff --git a/packages/eden-watcher/src/gql/queries/slot.gql b/packages/eden-watcher/src/gql/queries/slot.gql index 986d929e7..e9c94fb31 100644 --- a/packages/eden-watcher/src/gql/queries/slot.gql +++ b/packages/eden-watcher/src/gql/queries/slot.gql @@ -1,5 +1,5 @@ -query slot($id: String!, $blockHash: Bytes!){ - slot(id: $id, block: { hash: $blockHash }){ +query slot($id: String!, $block: Block_height){ + slot(id: $id, block: $block){ id owner delegate diff --git a/packages/eden-watcher/src/gql/queries/slotClaim.gql b/packages/eden-watcher/src/gql/queries/slotClaim.gql index faf631c0d..92ddf9ad8 100644 --- a/packages/eden-watcher/src/gql/queries/slotClaim.gql +++ b/packages/eden-watcher/src/gql/queries/slotClaim.gql @@ -1,5 +1,5 @@ -query slotClaim($id: String!, $blockHash: Bytes!){ - slotClaim(id: $id, block: { hash: $blockHash }){ +query slotClaim($id: String!, $block: Block_height){ + slotClaim(id: $id, block: $block){ id slot{ id diff --git a/packages/eden-watcher/src/gql/queries/staker.gql b/packages/eden-watcher/src/gql/queries/staker.gql index 89cbbf618..bb4cdd510 100644 --- a/packages/eden-watcher/src/gql/queries/staker.gql +++ b/packages/eden-watcher/src/gql/queries/staker.gql @@ -1,5 +1,5 @@ -query staker($id: String!, $blockHash: Bytes!){ - staker(id: $id, block: { hash: $blockHash }){ +query staker($id: String!, $block: Block_height){ + staker(id: $id, block: $block){ id staked rank diff --git a/packages/eden-watcher/src/hooks.ts b/packages/eden-watcher/src/hooks.ts index 9c2a95f14..b5b5cf6b4 100644 --- a/packages/eden-watcher/src/hooks.ts +++ b/packages/eden-watcher/src/hooks.ts @@ -2,7 +2,10 @@ // Copyright 2021 Vulcanize, Inc. // +import { IPLDBlockInterface, StateKind } from '@vulcanize/util'; import assert from 'assert'; +import * as codec from '@ipld/dag-cbor'; +import _ from 'lodash'; import { Indexer, ResultEvent } from './indexer'; @@ -51,9 +54,60 @@ export async function createStateCheckpoint (indexer: Indexer, contractAddress: assert(blockHash); assert(contractAddress); - // Use indexer.createStateCheckpoint() method to create a custom checkpoint. + // TODO: Pass blockProgress instead of blockHash to hook method. + const block = await indexer.getBlockProgress(blockHash); + assert(block); - return false; + // Fetch the latest 'checkpoint' | 'init' for the contract to fetch diffs after it. + let prevNonDiffBlock: IPLDBlockInterface; + let getDiffBlockNumber: number; + const checkpointBlock = await indexer.getLatestIPLDBlock(contractAddress, StateKind.Checkpoint, block.blockNumber); + + if (checkpointBlock) { + const checkpointBlockNumber = checkpointBlock.block.blockNumber; + + prevNonDiffBlock = checkpointBlock; + getDiffBlockNumber = checkpointBlockNumber; + + // Update IPLD status map with the latest checkpoint info. + // Essential while importing state as checkpoint at the snapshot block is added by import-state CLI. + // (job-runner won't have the updated ipld status) + indexer.updateIPLDStatusMap(contractAddress, { checkpoint: checkpointBlockNumber }); + } else { + // There should be an initial state at least. + const initBlock = await indexer.getLatestIPLDBlock(contractAddress, StateKind.Init); + assert(initBlock, 'No initial state found'); + + prevNonDiffBlock = initBlock; + // Take block number previous to initial state block to include any diff state at that block. + getDiffBlockNumber = initBlock.block.blockNumber - 1; + } + + // Fetching all diff blocks after the latest 'checkpoint' | 'init'. + const diffBlocks = await indexer.getDiffIPLDBlocksByBlocknumber(contractAddress, getDiffBlockNumber); + + const prevNonDiffBlockData = codec.decode(Buffer.from(prevNonDiffBlock.data)) as any; + const data = { + state: prevNonDiffBlockData.state + }; + + // Merge all diff blocks after previous checkpoint. + for (const diffBlock of diffBlocks) { + const diff = codec.decode(Buffer.from(diffBlock.data)) as any; + data.state = _.merge(data.state, diff.state); + } + + // Check if Block entity exists. + if (data.state.Block) { + // Store only block entity at checkpoint height instead of all entities. + data.state.Block = { + [blockHash]: data.state.Block[blockHash] + }; + } + + await indexer.createStateCheckpoint(contractAddress, blockHash, data); + + return true; } /** diff --git a/packages/eden-watcher/src/indexer.ts b/packages/eden-watcher/src/indexer.ts index bc65dd68e..0df201ddf 100644 --- a/packages/eden-watcher/src/indexer.ts +++ b/packages/eden-watcher/src/indexer.ts @@ -57,6 +57,7 @@ import { Account } from './entity/Account'; import { Slash } from './entity/Slash'; const log = debug('vulcanize:indexer'); +const JSONbigNative = JSONbig({ useNativeBigInt: true }); const KIND_EDENNETWORK = 'EdenNetwork'; const KIND_MERKLEDISTRIBUTOR = 'EdenNetworkDistribution'; @@ -176,8 +177,8 @@ export class Indexer implements IPLDIndexerInterface { getResultEvent (event: Event): ResultEvent { const block = event.block; - const eventFields = JSONbig.parse(event.eventInfo); - const { tx, eventSignature } = JSON.parse(event.extraInfo); + const eventFields = JSONbigNative.parse(event.eventInfo); + const { tx, eventSignature } = JSONbigNative.parse(event.extraInfo); return { block: { @@ -287,6 +288,10 @@ export class Indexer implements IPLDIndexerInterface { return this._baseIndexer.getIPLDBlockByCid(cid); } + async getDiffIPLDBlocksByBlocknumber (contractAddress: string, blockNumber: number): Promise { + return this._db.getDiffIPLDBlocksByBlocknumber(contractAddress, blockNumber); + } + getIPLDData (ipldBlock: IPLDBlock): any { return this._baseIndexer.getIPLDData(ipldBlock); } @@ -370,13 +375,15 @@ export class Indexer implements IPLDIndexerInterface { await this._baseIndexer.createInit(this, blockHash, blockNumber); console.timeEnd('time:indexer#processBlock-init_state'); + } - console.time('time:indexer#processBlock-mapping_code'); + async processBlockAfterEvents (blockHash: string): Promise { + console.time('time:indexer#processBlockAfterEvents-mapping_code'); // Call subgraph handler for block. await this._graphWatcher.handleBlock(blockHash); - console.timeEnd('time:indexer#processBlock-mapping_code'); + console.timeEnd('time:indexer#processBlockAfterEvents-mapping_code'); } parseEventNameAndArgs (kind: string, logObj: any): any { @@ -1013,10 +1020,10 @@ export class Indexer implements IPLDIndexerInterface { txHash, contract, eventName, - eventInfo: JSONbig.stringify(eventInfo), - extraInfo: JSONbig.stringify(extraInfo), - proof: JSONbig.stringify({ - data: JSONbig.stringify({ + eventInfo: JSONbigNative.stringify(eventInfo), + extraInfo: JSONbigNative.stringify(extraInfo), + proof: JSONbigNative.stringify({ + data: JSONbigNative.stringify({ blockHash, receiptCID, log: { diff --git a/packages/erc20-watcher/src/indexer.ts b/packages/erc20-watcher/src/indexer.ts index d1e9fc0b6..bf386b41e 100644 --- a/packages/erc20-watcher/src/indexer.ts +++ b/packages/erc20-watcher/src/indexer.ts @@ -23,6 +23,7 @@ import { BlockProgress } from './entity/BlockProgress'; import { Contract } from './entity/Contract'; const log = debug('vulcanize:indexer'); +const JSONbigNative = JSONbig({ useNativeBigInt: true }); const ETH_CALL_MODE = 'eth_call'; @@ -108,7 +109,7 @@ export class Indexer implements IndexerInterface { } // https://github.com/GoogleChromeLabs/jsbi/issues/30#issuecomment-521460510 - log(JSONbig.stringify(result, null, 2)); + log(JSONbigNative.stringify(result, null, 2)); return result; } @@ -142,10 +143,10 @@ export class Indexer implements IndexerInterface { result = await this._baseIndexer.getStorageValue(this._storageLayout, blockHash, token, '_balances', owner); } - log(JSONbig.stringify(result, null, 2)); + log(JSONbigNative.stringify(result, null, 2)); const { value, proof } = result; - await this._db.saveBalance({ blockHash, blockNumber, token, owner, value: BigInt(value), proof: JSONbig.stringify(proof) }); + await this._db.saveBalance({ blockHash, blockNumber, token, owner, value: BigInt(value), proof: JSONbigNative.stringify(proof) }); return result; } @@ -180,7 +181,7 @@ export class Indexer implements IndexerInterface { // log(JSONbig.stringify(result, null, 2)); const { value, proof } = result; - await this._db.saveAllowance({ blockHash, blockNumber, token, owner, spender, value: BigInt(value), proof: JSONbig.stringify(proof) }); + await this._db.saveAllowance({ blockHash, blockNumber, token, owner, spender, value: BigInt(value), proof: JSONbigNative.stringify(proof) }); return result; } @@ -406,10 +407,10 @@ export class Indexer implements IndexerInterface { txHash, contract, eventName, - eventInfo: JSONbig.stringify(eventInfo), - extraInfo: JSONbig.stringify(extraInfo), - proof: JSONbig.stringify({ - data: JSONbig.stringify({ + eventInfo: JSONbigNative.stringify(eventInfo), + extraInfo: JSONbigNative.stringify(extraInfo), + proof: JSONbigNative.stringify({ + data: JSONbigNative.stringify({ blockHash, receiptCID, log: { diff --git a/packages/erc721-watcher/src/indexer.ts b/packages/erc721-watcher/src/indexer.ts index 9a25405b7..0cd0813e0 100644 --- a/packages/erc721-watcher/src/indexer.ts +++ b/packages/erc721-watcher/src/indexer.ts @@ -42,6 +42,7 @@ import { IPLDBlock } from './entity/IPLDBlock'; import { TransferCount } from './entity/TransferCount'; const log = debug('vulcanize:indexer'); +const JSONbigNative = JSONbig({ useNativeBigInt: true }); const KIND_ERC721 = 'ERC721'; @@ -134,8 +135,8 @@ export class Indexer implements IPLDIndexerInterface { getResultEvent (event: Event): ResultEvent { const block = event.block; - const eventFields = JSONbig.parse(event.eventInfo); - const { tx, eventSignature } = JSON.parse(event.extraInfo); + const eventFields = JSONbigNative.parse(event.eventInfo); + const { tx, eventSignature } = JSONbigNative.parse(event.extraInfo); return { block: { @@ -211,7 +212,7 @@ export class Indexer implements IPLDIndexerInterface { const result: ValueResult = { value }; - await this._db.saveSupportsInterface({ blockHash, blockNumber, contractAddress, interfaceId, value: result.value, proof: JSONbig.stringify(result.proof) }); + await this._db.saveSupportsInterface({ blockHash, blockNumber, contractAddress, interfaceId, value: result.value, proof: JSONbigNative.stringify(result.proof) }); return result; } @@ -242,7 +243,7 @@ export class Indexer implements IPLDIndexerInterface { const result: ValueResult = { value }; - await this._db.saveBalanceOf({ blockHash, blockNumber, contractAddress, owner, value: result.value, proof: JSONbig.stringify(result.proof) }); + await this._db.saveBalanceOf({ blockHash, blockNumber, contractAddress, owner, value: result.value, proof: JSONbigNative.stringify(result.proof) }); return result; } @@ -271,7 +272,7 @@ export class Indexer implements IPLDIndexerInterface { const result: ValueResult = { value }; - await this._db.saveOwnerOf({ blockHash, blockNumber, contractAddress, tokenId, value: result.value, proof: JSONbig.stringify(result.proof) }); + await this._db.saveOwnerOf({ blockHash, blockNumber, contractAddress, tokenId, value: result.value, proof: JSONbigNative.stringify(result.proof) }); return result; } @@ -300,7 +301,7 @@ export class Indexer implements IPLDIndexerInterface { const result: ValueResult = { value }; - await this._db.saveGetApproved({ blockHash, blockNumber, contractAddress, tokenId, value: result.value, proof: JSONbig.stringify(result.proof) }); + await this._db.saveGetApproved({ blockHash, blockNumber, contractAddress, tokenId, value: result.value, proof: JSONbigNative.stringify(result.proof) }); return result; } @@ -329,7 +330,7 @@ export class Indexer implements IPLDIndexerInterface { const result: ValueResult = { value }; - await this._db.saveIsApprovedForAll({ blockHash, blockNumber, contractAddress, owner, operator, value: result.value, proof: JSONbig.stringify(result.proof) }); + await this._db.saveIsApprovedForAll({ blockHash, blockNumber, contractAddress, owner, operator, value: result.value, proof: JSONbigNative.stringify(result.proof) }); return result; } @@ -358,7 +359,7 @@ export class Indexer implements IPLDIndexerInterface { const result: ValueResult = { value }; - await this._db.saveName({ blockHash, blockNumber, contractAddress, value: result.value, proof: JSONbig.stringify(result.proof) }); + await this._db.saveName({ blockHash, blockNumber, contractAddress, value: result.value, proof: JSONbigNative.stringify(result.proof) }); return result; } @@ -387,7 +388,7 @@ export class Indexer implements IPLDIndexerInterface { const result: ValueResult = { value }; - await this._db.saveSymbol({ blockHash, blockNumber, contractAddress, value: result.value, proof: JSONbig.stringify(result.proof) }); + await this._db.saveSymbol({ blockHash, blockNumber, contractAddress, value: result.value, proof: JSONbigNative.stringify(result.proof) }); return result; } @@ -416,7 +417,7 @@ export class Indexer implements IPLDIndexerInterface { const result: ValueResult = { value }; - await this._db.saveTokenURI({ blockHash, blockNumber, contractAddress, tokenId, value: result.value, proof: JSONbig.stringify(result.proof) }); + await this._db.saveTokenURI({ blockHash, blockNumber, contractAddress, tokenId, value: result.value, proof: JSONbigNative.stringify(result.proof) }); return result; } @@ -477,7 +478,7 @@ export class Indexer implements IPLDIndexerInterface { '_name' ); - await this._db._saveName({ blockHash, blockNumber, contractAddress, value: result.value, proof: JSONbig.stringify(result.proof) }); + await this._db._saveName({ blockHash, blockNumber, contractAddress, value: result.value, proof: JSONbigNative.stringify(result.proof) }); if (diff) { const stateUpdate = updateStateForElementaryType({}, '_name', result.value.toString()); @@ -513,7 +514,7 @@ export class Indexer implements IPLDIndexerInterface { '_symbol' ); - await this._db._saveSymbol({ blockHash, blockNumber, contractAddress, value: result.value, proof: JSONbig.stringify(result.proof) }); + await this._db._saveSymbol({ blockHash, blockNumber, contractAddress, value: result.value, proof: JSONbigNative.stringify(result.proof) }); if (diff) { const stateUpdate = updateStateForElementaryType({}, '_symbol', result.value.toString()); @@ -550,7 +551,7 @@ export class Indexer implements IPLDIndexerInterface { key0 ); - await this._db._saveOwners({ blockHash, blockNumber, contractAddress, key0, value: result.value, proof: JSONbig.stringify(result.proof) }); + await this._db._saveOwners({ blockHash, blockNumber, contractAddress, key0, value: result.value, proof: JSONbigNative.stringify(result.proof) }); if (diff) { const stateUpdate = updateStateForMappingType({}, '_owners', [key0.toString()], result.value.toString()); @@ -587,7 +588,7 @@ export class Indexer implements IPLDIndexerInterface { key0 ); - await this._db._saveBalances({ blockHash, blockNumber, contractAddress, key0, value: result.value, proof: JSONbig.stringify(result.proof) }); + await this._db._saveBalances({ blockHash, blockNumber, contractAddress, key0, value: result.value, proof: JSONbigNative.stringify(result.proof) }); if (diff) { const stateUpdate = updateStateForMappingType({}, '_balances', [key0.toString()], result.value.toString()); @@ -624,7 +625,7 @@ export class Indexer implements IPLDIndexerInterface { key0 ); - await this._db._saveTokenApprovals({ blockHash, blockNumber, contractAddress, key0, value: result.value, proof: JSONbig.stringify(result.proof) }); + await this._db._saveTokenApprovals({ blockHash, blockNumber, contractAddress, key0, value: result.value, proof: JSONbigNative.stringify(result.proof) }); if (diff) { const stateUpdate = updateStateForMappingType({}, '_tokenApprovals', [key0.toString()], result.value.toString()); @@ -662,7 +663,7 @@ export class Indexer implements IPLDIndexerInterface { key1 ); - await this._db._saveOperatorApprovals({ blockHash, blockNumber, contractAddress, key0, key1, value: result.value, proof: JSONbig.stringify(result.proof) }); + await this._db._saveOperatorApprovals({ blockHash, blockNumber, contractAddress, key0, key1, value: result.value, proof: JSONbigNative.stringify(result.proof) }); if (diff) { const stateUpdate = updateStateForMappingType({}, '_operatorApprovals', [key0.toString(), key1.toString()], result.value.toString()); @@ -1057,10 +1058,10 @@ export class Indexer implements IPLDIndexerInterface { txHash, contract, eventName, - eventInfo: JSONbig.stringify(eventInfo), - extraInfo: JSONbig.stringify(extraInfo), - proof: JSONbig.stringify({ - data: JSONbig.stringify({ + eventInfo: JSONbigNative.stringify(eventInfo), + extraInfo: JSONbigNative.stringify(extraInfo), + proof: JSONbigNative.stringify({ + data: JSONbigNative.stringify({ blockHash, receiptCID, log: { diff --git a/packages/graph-node/README.md b/packages/graph-node/README.md index 18cabb41e..66f08124e 100644 --- a/packages/graph-node/README.md +++ b/packages/graph-node/README.md @@ -55,3 +55,46 @@ ``` * The program will exit with code `1` if the query results are not equal. + + * For comparing queries in a range of blocks: + + * Config file should have the names of queries to be fired. + + ```toml + [queries] + queryDir = "../graph-test-watcher/src/gql/queries" + names = [ + "author", + "blog" + ] + ``` + + * Run the CLI: + + ```bash + ./bin/compare-blocks --config-file environments/compare-cli-config.toml --start-block 1 --end-block 10 + ``` + + * For comparing entities after fetching ids from one of the endpoints and then querying individually by ids: + + * Set the `idsEndpoint` to choose which endpoint the ids should be fetched from. + + ```toml + [endpoints] + gqlEndpoint1 = "http://localhost:8000/subgraphs/name/example1" + gqlEndpoint2 = "http://localhost:3008/graphql" + + [queries] + queryDir = "../graph-test-watcher/src/gql/queries" + names = [ + "author", + "blog" + ] + idsEndpoint = "gqlEndpoint1" + ``` + + * Run the CLI with `fetch-ids` flag set to true:\ + + ```bash + ./bin/compare-blocks --config-file environments/compare-cli-config.toml --start-block 1 --end-block 10 --fetch-ids + ``` diff --git a/packages/graph-node/environments/compare-cli-config.toml b/packages/graph-node/environments/compare-cli-config.toml index 78dd811ff..0cf851ac9 100644 --- a/packages/graph-node/environments/compare-cli-config.toml +++ b/packages/graph-node/environments/compare-cli-config.toml @@ -5,6 +5,7 @@ [queries] queryDir = "../graph-test-watcher/src/gql/queries" names = [] + idsEndpoint = "gqlEndpoint1" [cache] endpoint = "gqlEndpoint1" diff --git a/packages/graph-node/package.json b/packages/graph-node/package.json index 5b10d4711..feea870bd 100644 --- a/packages/graph-node/package.json +++ b/packages/graph-node/package.json @@ -10,6 +10,7 @@ "@types/chai": "^4.2.18", "@types/chai-spies": "^1.0.3", "@types/js-yaml": "^4.0.4", + "@types/pluralize": "^0.0.29", "@typescript-eslint/eslint-plugin": "^4.25.0", "@typescript-eslint/parser": "^4.25.0", "chai": "^4.3.4", @@ -57,6 +58,7 @@ "js-yaml": "^4.1.0", "json-bigint": "^1.0.0", "json-diff": "^0.5.4", + "pluralize": "^8.0.0", "reflect-metadata": "^0.1.13", "toml": "^3.0.0", "typeorm": "^0.2.32", diff --git a/packages/graph-node/src/cli/compare/client.ts b/packages/graph-node/src/cli/compare/client.ts index 4d1121872..4b8af9c90 100644 --- a/packages/graph-node/src/cli/compare/client.ts +++ b/packages/graph-node/src/cli/compare/client.ts @@ -5,6 +5,7 @@ import assert from 'assert'; import fs from 'fs'; import path from 'path'; +import pluralize from 'pluralize'; import { gql } from '@apollo/client/core'; import { GraphQLClient, Config } from '@vulcanize/ipld-eth-client'; @@ -15,6 +16,7 @@ export class Client { _graphqlClient: GraphQLClient; _queryDir: string; _cache: Cache | undefined; + _endpoint: string; constructor (config: Config, queryDir: string) { this._config = config; @@ -22,16 +24,56 @@ export class Client { const { gqlEndpoint, cache } = config; assert(gqlEndpoint, 'Missing gql endpoint'); + this._endpoint = gqlEndpoint; this._graphqlClient = new GraphQLClient(config); this._cache = cache; } + get endpoint () { + return this._endpoint; + } + async getResult (queryName: string, params: { [key: string]: any }): Promise { return this._getCachedOrFetch(queryName, params); } + async getIds (queryName: string, blockNumber: number): Promise { + const keyObj = { queryName, blockNumber }; + + if (this._cache) { + const [value, found] = await this._cache.get(keyObj) || [undefined, false]; + if (found) { + return value; + } + } + + const result = await this._graphqlClient.query( + gql( + `query($blockNumber: Int){ + ${pluralize(queryName)}( + block: { number: $blockNumber } + ) { + id + } + }` + ), + { + blockNumber + } + ); + + const ids = result[pluralize(queryName)].map((entity: { id: string }) => entity.id); + + // Cache the result and return it, if cache is enabled. + if (this._cache) { + await this._cache.put(keyObj, ids); + } + + return ids; + } + async _getCachedOrFetch (queryName: string, params: {[key: string]: any}): Promise { const keyObj = { queryName, diff --git a/packages/graph-node/src/cli/compare/compare-blocks.ts b/packages/graph-node/src/cli/compare/compare-blocks.ts index d205328bc..77fd85cb3 100644 --- a/packages/graph-node/src/cli/compare/compare-blocks.ts +++ b/packages/graph-node/src/cli/compare/compare-blocks.ts @@ -5,8 +5,10 @@ import yargs from 'yargs'; import 'reflect-metadata'; import debug from 'debug'; +import assert from 'assert'; import { compareQuery, Config, getClients, getConfig } from './utils'; +import { Client } from './client'; const log = debug('vulcanize:compare-blocks'); @@ -40,12 +42,17 @@ export const main = async (): Promise => { type: 'boolean', describe: 'Whether to print out raw diff object', default: false + }, + fetchIds: { + type: 'boolean', + describe: 'Fetch ids and compare multiple entities', + default: false } }).argv; const config: Config = await getConfig(argv.configFile); - const { startBlock, endBlock, rawJson, queryDir } = argv; + const { startBlock, endBlock, rawJson, queryDir, fetchIds } = argv; const queryNames = config.queries.names; let diffFound = false; @@ -58,13 +65,27 @@ export const main = async (): Promise => { for (const queryName of queryNames) { try { log(`At block ${blockNumber} for query ${queryName}:`); - const resultDiff = await compareQuery(clients, queryName, { block }, rawJson); - if (resultDiff) { - diffFound = true; - log('Results mismatch:', resultDiff); + if (fetchIds) { + const { idsEndpoint } = config.queries; + assert(idsEndpoint, 'Specify endpoint for fetching ids when fetchId is true'); + const client = Object.values(clients).find(client => client.endpoint === config.endpoints[idsEndpoint]); + assert(client); + const ids = await client.getIds(queryName, blockNumber); + + for (const id of ids) { + const isDiff = await compareAndLog(clients, queryName, { block, id }, rawJson); + + if (isDiff) { + diffFound = isDiff; + } + } } else { - log('Results match.'); + const isDiff = await compareAndLog(clients, queryName, { block }, rawJson); + + if (isDiff) { + diffFound = isDiff; + } } } catch (err: any) { log('Error:', err.message); @@ -78,3 +99,25 @@ export const main = async (): Promise => { process.exit(1); } }; + +const compareAndLog = async ( + clients: { client1: Client, client2: Client }, + queryName: string, + params: { [key: string]: any }, + rawJson: boolean +): Promise => { + const resultDiff = await compareQuery( + clients, + queryName, + params, + rawJson + ); + + if (resultDiff) { + log('Results mismatch:', resultDiff); + return true; + } + + log('Results match.'); + return false; +}; diff --git a/packages/graph-node/src/cli/compare/compare-entity.ts b/packages/graph-node/src/cli/compare/compare-entity.ts index 8af460239..18c570f29 100644 --- a/packages/graph-node/src/cli/compare/compare-entity.ts +++ b/packages/graph-node/src/cli/compare/compare-entity.ts @@ -54,11 +54,10 @@ export const main = async (): Promise => { const queryName = argv.queryName; const id = argv.entityId; - const blockHash = argv.blockHash; const block = { number: argv.blockNumber, - hash: blockHash + hash: argv.blockHash }; const clients = await getClients(config, argv.queryDir); diff --git a/packages/graph-node/src/cli/compare/utils.ts b/packages/graph-node/src/cli/compare/utils.ts index 8754c5319..f77ff7880 100644 --- a/packages/graph-node/src/cli/compare/utils.ts +++ b/packages/graph-node/src/cli/compare/utils.ts @@ -21,13 +21,14 @@ interface EndpointConfig { interface QueryConfig { queryDir: string; names: string[]; + idsEndpoint: keyof EndpointConfig; } export interface Config { endpoints: EndpointConfig; queries: QueryConfig; cache: { - endpoint: string; + endpoint: keyof EndpointConfig; config: CacheConfig; } } diff --git a/packages/graph-node/src/database.ts b/packages/graph-node/src/database.ts index cf053e49f..f2b192b0d 100644 --- a/packages/graph-node/src/database.ts +++ b/packages/graph-node/src/database.ts @@ -17,6 +17,8 @@ import { import { Block, fromEntityValue, toEntityValue } from './utils'; +const DEFAULT_LIMIT = 100; + export class Database { _config: ConnectionOptions _conn!: Connection @@ -123,7 +125,8 @@ export class Database { if (isArray) { selectQueryBuilder = selectQueryBuilder.distinctOn(['entity.id']) - .orderBy('entity.id'); + .orderBy('entity.id') + .limit(DEFAULT_LIMIT); } else { selectQueryBuilder = selectQueryBuilder.limit(1); } @@ -132,7 +135,8 @@ export class Database { // For one to many relational field. selectQueryBuilder = selectQueryBuilder.where('entity.id IN (:...ids)', { ids: entityData[field] }) .distinctOn(['entity.id']) - .orderBy('entity.id'); + .orderBy('entity.id') + .limit(DEFAULT_LIMIT); // Subquery example if distinctOn is not performant. // diff --git a/packages/graph-node/src/loader.ts b/packages/graph-node/src/loader.ts index f1f4b4e14..a0e58d314 100644 --- a/packages/graph-node/src/loader.ts +++ b/packages/graph-node/src/loader.ts @@ -29,6 +29,8 @@ import { } from './utils'; import { Database } from './database'; +const JSONbigString = JSONbig({ storeAsString: true }); + // Endianness of BN used in bigInt store host API. // Negative bigInt is being stored in wasm in 2's compliment, 'le' representation. // (for eg. bigInt.fromString(negativeI32Value)) @@ -104,7 +106,12 @@ export const instantiate = async ( // JSON stringify and parse data for handling unknown types when encoding. // For example, decimal.js values are converted to string in the diff data. - diffData.state[entityName] = JSONbig.parse(JSONbig.stringify(dbData)); + diffData.state[entityName] = { + // Using JSONbigString to store bigints as string values to be encoded by IPLD dag-cbor. + // TODO: Parse and store as native bigint by using Type encoders in IPLD dag-cbor encode. + // https://github.com/rvagg/cborg#type-encoders + [dbData.id]: JSONbigString.parse(JSONbigString.stringify(dbData)) + }; // Create an auto-diff. assert(indexer.createDiffStaged); diff --git a/packages/graph-node/src/watcher.ts b/packages/graph-node/src/watcher.ts index dba426092..b63845489 100644 --- a/packages/graph-node/src/watcher.ts +++ b/packages/graph-node/src/watcher.ts @@ -123,7 +123,7 @@ export class GraphWatcher { async handleEvent (eventData: any) { const { contract, event, eventSignature, block, tx: { hash: txHash }, eventIndex } = eventData; - if (!this._context.block) { + if (!this._context.block || this._context.block.blockHash !== block.hash) { this._context.block = await getFullBlock(this._ethClient, this._ethProvider, block.hash); } diff --git a/packages/graph-node/test/subgraph/eden/EdenNetwork/EdenNetwork.wasm b/packages/graph-node/test/subgraph/eden/EdenNetwork/EdenNetwork.wasm index 8a084cc56f203451c620ff71e1f9de4e24b91768..612a9fd170e3603bafb4f216a79f9fc09657771e 100644 GIT binary patch delta 37 tcmZqL&Dpk_bAz%uE=4~q(+c&R%(8a-`%&o%LZ2EG$=}Si65&$1G4!r;X delta 42 ycmZqM&Dpe@bAz%ulvqiiCp;98m#Bb<;_2AVt+(yij#lclPVGyt^@-YPXRJo& zah==T8K%GXXVui=bln-ciVm|WX!MyHCQP5H&}R)=BlKBXs760Q10j8#NC1Bltr7YhEmWh=)j&w^SLt~ydu#BW6M3~WsvowGL~EK>6pcq zn^>q}c*>%P-zbBUvPel;9I>k@i+RddrfSy5Qcl(Aw3HnmKj(=nF`W#E@i^qf`*XYGo(yCK7u;gEod-> zWPb!Bnb2Q^Jo~BR>$G%Puo@&2KU07wJ1~d?c?6Eset`zEVkcp3N)%_)FUzarM?=O@j`>vKY-x0$qP=6kS*!8}AwP86>c*kUexm{n#fD zN6a#bH0*G=7*LA|cL&>&$a~xz3Blw%OEc}Rw@y70G0SysT(IR?bRBt|Ntqn)AWVnx z2H8ueeKTrW%b;1nI&25kt!w$W_|`$7f4_hX&(bB|NaIwl?;>!e%f1L!ingqjeFFXS z=8|YbAw)|=($f*&wt>HyUi-EV+%ue?{_~_BY#q*D9ZDu(Yops6Dxj{74sEOee;fS` zipqA{-?$Cj?evdLvpA%!={Zwf8!w_*9!)oRM$uWGi5&e)&+{DJ?0Fy2&ZiF(0@WS# zndaF#+X;bAUzkDnH~&Nz#DZ4rp67h^ND_eCM>n+0;lkJQl@3{zuh;{veMDE;t|;`T z@2b&%T9Ex+wEO4JISQs zB-1(a#6}%)eO7e}pt^y6+=Xjxps{Bvz}84Nqo`}7eJK2mbm>_%Hqo;v>YC^a=PDrB zL>p05duWNj0^A*IG1O!U7XLeoV-}hvrM{F&a?b43e0jD1!j>i zuK{;6{r}5zQ8;6IM`7O^==duOhSnWX4yw=f%1PHoNk;uEg0M%EMQ!HmC7^K2A(KEgmS~O;|SY z0o3IL%-{uk^Bc1ljHhvG-=Rr zq!@!9WcXbSSpe>ftUHE04|SK=Vzs&v?OK`I z^~Sm@d^BBShLcA|qwgwLm~{-fZt{Px&g^kzW?k}FLX0NcHP!f#8%Md`NF%!?d{=ZO zJW0w8V&!$!`=L7et{t<vs^^0|+U1tj3#qk;< z-xru|6Pf4w@I5kK2f@3pee20R5*_w#Wjp6#sqGucUqG|%Wc$~X7hUa}NSYGx;J0M6 zaD7bpp6mZw z$mnRW-Dd~dajDh!*_C#j1Y17)eFu3F{P$gFJIGtH(ZL4_Pg*1yt$H<+RbC~lU1gVu z$qeoR*X*0*5OVgrEPbRP8Y*wGS&xX7cYg6BvJ3nJZ1@l4Dgw3vw1MNddphCgM7M4A zMOPufHp_%;K@Me`M-f2~;_+G|tQ8gs75I0q@L$4m;Y~qyT zV1G@3F(U(LLyAzI*~BRTuY3XjeR7>nfUQJyOfo}~9t)V{dT%(y=uP$0%;aT3_Do}c zOoX4A9HmNzno@Q(5!P57)6K+==;`M7IomtzW+J@A+(xjPYG%JpZ!r0SlO~97;5sO=NVji8T)+_Ocxz)!Axo~lbf|9!KcH# z`%!nGALSnikKN2>=D}$8@fdhJF>nCAn9Y6=<$7!w$~QS^Auq6qy^;aZY~ff)743(@ zh(qjyvGA^FZ$P~iBO2I{WXR9(avhmEQ4Zw{yznKsYisbY9N=Euu22JeGZ`e&(WvS* zjqFdTXM4HMqfth7Y>}`IcYKMk5_et>gd;+Y>~u1W6&+0&f%$8im`;Lhv5V`Np|^=m zl^|`7-J>#Ap^PxDK*r_5Yd8bQSj!O@u@~i!c*U`JEmkU?hgH9cvm$qeF})tPOM;{V z`(ZVv3T0$E*M?_OozmsPDnYIf<2;Ai1AGS+16;>yLx)+q6|(Yco7L*}qKtHF4f6bo z&$gF)pTTaRoWd*1#qCZVwGunA4E>96j&tz|ID)8`u>(igJ}Z=q16*Gw$XmQ@ObWb} z74WJ#LMY=ni-^|~$gom)U3eS!yaoO1aR*!2t`w*!u(zuC+OXV<(v*WSd?x{`3flu* zI&dwm?D}yz`3Z$70tBG-83hC)i8lL5jC`s-P;AKj+&qm#?;n9T*Q*aezB(u@@)V zwej$|SbGX}2Ud5ARcFATMek|U*W*N-X45k9>GO7}WSw2?n@m_D)}KM$j;GHV7FPht z?71v>UG$z+jh$!NzAV@vcAZo8-gC@05hQjh3(vY=PJo}q2mC5s2<3cU+Z=T#;<6!G z?CM4*Ur0AA$%Z7+ejatKwdy>3H5)z?>o2H&&jmIl2NJ~q*Lf!|u!%V^T(tM7{;D4K zd=8|E^IWD1&Yp=P)evu3Bh>puD zU(IE9E*Hj$-mu1;x zcYe?P67Ster7dD0pdv#7-mw1`2d=65mg~IBn??H#HL~gk`=kK=B-RE|e;F$evcgI5 zqG-Q~I)eJ8an$n96FRAWj?z8@Jr$#@S-mT{TnhUFM#GJ@fXVo{c1Zte}ujKi9FxTl(1> zg)lj#z901@>;cNi*Bj!yxE*o5{p@SRiH;EJSWHcbg$iM&7zpbjMbBh!7r_Xz{vLub z*mIBBi}(O=9mzxYSX~i}6z%s_(yIIHSP^8Uc)5=LPLz@4_@x{O<3jh@*kZ^RYagh* zdmpg!Vq8QQ*Rh!12kd`}q4cTR0hMPjN>fGnpwi(%6}ESB`d%|M$d;XVHV{x}olAd9Li?L{$gD6L#o$ZT#Ki{>nr~SNEEQu delta 5990 zcmY*d3s_WFlD?<<0eu@ti*G<_9wP6TplCq7Ni^dl84@+RKC)^iiBY2ln2a-P6rD-J zOqfkOC+3(*Of@r|BZ*WjSqXM?Nb?)uk82m2PRrS|j zb?VeTr{&`x#l~jQc25sg5A65#YqR0tHv5_h8Lvh@2?NzNR)XJd0cc`_n~W%cQZ#$J;0W_hqDCfj@b1Wo8ATkax%B)6VB5#(dD2_;Gn)h`Q*mCINuiqLR7xe2SVltyU zj&^;WHa|XSk1*@xuy`erCdy3WLX1eAph6_Alq82HJOa^iNTLcsjx^*>q-(y3S~y0v zN`fSqO{NU=jCmwf%R`e?kXa|WBFtJ-fs~|TjAj#TM9f$fBMGK4NHUhb=oq;$dB7@# z1n861P%K`iPfU4^r3s@)x{6WhGgM4~ zK7-S5${er;=`+<(l|D-aA-yF_u0Bh}@aAO){dy!%#@1x3ZiPbWUrrFFH7!9Thu4=~(QahOYYXXflP6Z8gaGZ+aHRJ{TdYmj?L!Rn>w1)g3OsOG1P{VlD{zwhu ze@&o~Ca9o+HcwD7yoQ3HUoU}NLxJjkw1$a6m{P+;rG_PwRQn?}O!_r}MtWQY4eZ6^ zD#mEm;vt}s3Ln9^B20zYjKY6>8#kgzwM)JtscMk)Pmpx@P8ZRM-xh3|Jm8!x`6A_) zaI(Uz<1eAWS2#rlnFUU;8>A21J5)_!0v&nBoZ4e@Lr%f?y z@wL@RwLuu3EgI=S5Mwr=e*zL6pd-Ia9Ah@%Nf)P+1b%dB@SF{%(@Dc|3}+$oAYJ}l zdW3Ff98#M@%-U!(%w$VHCqoF(ufB^JV{`FSQaNAh1_+5-M2|u=E1yFnxaC*dg@;9)nphy^i7a^zZxP zBHaTH!SpJc-PE-&IkI8EtTTgYjnqK<_l+Lb7&J>HxZp;b=^Pb#BN@ZSOT_YI{Z)x_?qc^C@d}XAqrOuEj_#qx?8AN z{}hK5*FVc4RrSwt$So8IneKiIA@)OVLcn{9x*KL{Y^}?Qcn{NIZMdp>`nR?c>Ny)q zwe73HS5HgY(dwp`QP|vc;aO~en;t_^)j*&4xdglo^#4$FH_%ic8XM_O6y8P}d9DP? zkI;8e*pATtbDtY5^#aaYfs{jg<_)IK^ONK>56)x1>glr==JGV3p};i#C@@X&#koAq z=O{3Z|Kfa}X5OU*Jk6IVFpc5z0-ok26qx4w%canLgyvqEi^5)UWhA!yFx}O$5PTl` zumktvD1E;3O|TuKjh)rtJx7 zwd7-t+oVG~;`(&tkLY)l>B(=fxGcy15OPSvk#8H(q$E8a+qj;je|ml?kvI^KVZEYRs)H)Q;^8KhHZ@OCTs zYW_YrLW@b5F0{J`Z=P_;{b2=(a*Urx21SGSCY!O2tOVCBE+1#z>&OcBu#|+5B6hA! zJ|rfweQ%IIgj#OP!3UVHj4X6~P(oG_@ZDyljGO}39mkn6(yW21yN<8clRs-htM1C^ zcQMrqj(=<*J8&lMI&wCWT2OtT%D&>wBryp8LN=ZF3rUix`rjr0WiVQLaV19SV~`># zSry~hx0U!bVCi-2*hcOQ0aq{E_P?Z3U*0E}{DhxmjyHFc_MxFweez*9kso%~|7XSl z0^sX&%&H^50NDB+Qax$cLHRw0`6!7F1=~GV)J(>J_a6IQGY*0+hmjWY0$BWxe`_JH zL_pPjN9{#|*I4&`$DwXgr-kx9$9uO)ZYXcw5BEu2xc4T0OXmOZUMI}8j%2e@4+tJ9 z57^cR4`!?>P)@%g>5MtE6RFRa7g zF9~J%`*(OJ{2uj}gk?f0TBMo`BUvp-dozs7c+^6+UkAg7PD9|!82l{$E))JBtQAUx zRm_|KLrIn_Oqpme6(nn>k)#V!btan?38M@iMQzPwZ$!cvgCA8asUnMg5(y~=XO@w~ z2$DOCHATX3LmR3a1gRs7^+dv2LtC~C?#O1Z4ThX(YmN-A&M_KzVSrDzfdAn$zOP}6 zhC;HVeK5R6L{FX(Vzh$Pn&&7U3SnA9SD}$C#x@kPmq)@JgQrMN(pto7M?$H=GucQg z5j~j|MZso6+Y}||6jmPvISE#mpd>|UNX3ky_z=v>?YI!POJIWxki?3jVUW?!t+R1? zwE~+T4Mn1-77aDXUdwhy!zY9N2T->n;GkgqE8(ly!UknRG+Pw|t0Jv+=#4>gE;k@~ z9qWjJm7?=dKtIG5kAn50^Dyc>`C)c;6l5p+xsGua^@8y+PRfg1A@Ao3+z_o7hBHxy9DJK4D|%!GH>(*9qeTzbF@LL@og5A6 zVr7Hk-`>E)SV&BFavd4nD1(e^knwe4IgSr9wgn{J zg)w3gCJhnI{(4W<8=*R(*usiG|ecHm+kO9VmmHDMg+? zA&=zeUcOx(!I;mpWU{7wNEo>mm%SPR%W&Q^1<8YMK0vJ=R%wO>V$D(1SK}LUlwCK& zE28z7lBD_=dod1%iB7I>z~vuf8{?qlap!Tx-xiP^C=D5yj&BoS83DW1i#RL3Iw=1x z563%vIP%%jc$gREYznY&nT|_B8Ow_v!OS{{&XX!nD1hxK2XTjp&ou1PT70*04r;jH zgJq-qMyCHlp`VigZ$Ge?YHjVO5EcoLkeP;P;{oZs#)W@oM1-e0YJY_}S6MafP_H7S^8#x#`wZ zO1^58!9hyHoe3_=$q`tt`xM`s1lB$V;@Ol@Fj%ylfi0r{XGN{>vC0(KC{~_R^zG-^bJ<{K zyHj8Y`%Nm$iL{H}SBRaW`Rk|;*oHW=ac3o8b zy%*WpG>A&IUP3(v`+zby7LRt2`;pImiN&WwqSzJCdoQu#bVwGhm(gE{@t4__bjTIk zxV{>@aGC8-$5nc+ppIQ=y~46H;7!rfq42eKFk1#BiT;3I(aC%nI4d=s=(l25J6UKZ zyeE3DDthZxwgdIZu79CkiCshaAAIp;Y*Z=?XKBe0!Tvo9=7|0-C8FXQk1(^?6c`~l zPPATEVymz7mcA?2+)(trH*g1H+4q?UoR|!=L}#}`;qGQ@a^QWjtq1kRxTqc$k_#`0 z&YP&?BHcIHtGTdTthuG=dvCEHb74iK^%n(Sjq+(;WUY4H^dGTFbkHIdX4Bm!_uk2HB+xysih!Z_r$68wZSVKNc6|Mb>Ufs|26gDLq z^t^Ww;dsz>aVL`W_Om79@l>|nLmiV;-(#D{!wAvIbu7YtkL?-{sqy}Rzrz1W5$o}I zKwTagKf5s=vc)#8BX5VFrA)xZRNhA&i`sslEt&v@)7!X?c{)%UO0b*!VASBj7_e77 zKznJxjyF608PynGhPoLa{)2z_a6Vw|6Ci%mMedW(X26F_9X92mU=+>xng0NOjjBRJ zxJl9Fs|5+_&<5PuiXVn?Co1w!P#Ul`=rpL2A^7#NoEQnpDwHG9&Nddn2&oP=g)3nG z6<11tJ5i9zHxr{?&b18NfeOJdc-5N;J5>PBfPXVfng~xX^lS;xwW%LE@B^8F9YQGv v>%-zd_%k|iH8m)Q^7^00hs!wuiS6^FPp;1T7gj$J*1~6RI|?SjOKblh3hdnR diff --git a/packages/graph-test-watcher/src/indexer.ts b/packages/graph-test-watcher/src/indexer.ts index fc538515e..e818ccef5 100644 --- a/packages/graph-test-watcher/src/indexer.ts +++ b/packages/graph-test-watcher/src/indexer.ts @@ -46,6 +46,7 @@ import { Blog } from './entity/Blog'; import { Category } from './entity/Category'; const log = debug('vulcanize:indexer'); +const JSONbigNative = JSONbig({ useNativeBigInt: true }); const KIND_EXAMPLE1 = 'Example1'; const KIND_FACTORY = 'Factory'; @@ -167,8 +168,8 @@ export class Indexer implements IPLDIndexerInterface { getResultEvent (event: Event): ResultEvent { const block = event.block; - const eventFields = JSONbig.parse(event.eventInfo); - const { tx, eventSignature } = JSON.parse(event.extraInfo); + const eventFields = JSONbigNative.parse(event.eventInfo); + const { tx, eventSignature } = JSONbigNative.parse(event.extraInfo); return { block: { @@ -244,7 +245,7 @@ export class Indexer implements IPLDIndexerInterface { const result: ValueResult = { value }; - await this._db.saveGetMethod({ blockHash, blockNumber, contractAddress, value: result.value, proof: JSONbig.stringify(result.proof) }); + await this._db.saveGetMethod({ blockHash, blockNumber, contractAddress, value: result.value, proof: JSONbigNative.stringify(result.proof) }); return result; } @@ -275,7 +276,7 @@ export class Indexer implements IPLDIndexerInterface { '_test' ); - await this._db._saveTest({ blockHash, blockNumber, contractAddress, value: result.value, proof: JSONbig.stringify(result.proof) }); + await this._db._saveTest({ blockHash, blockNumber, contractAddress, value: result.value, proof: JSONbigNative.stringify(result.proof) }); if (diff) { const stateUpdate = updateStateForElementaryType({}, '_test', result.value.toString()); @@ -406,7 +407,9 @@ export class Indexer implements IPLDIndexerInterface { async processBlock (blockHash: string, blockNumber: number): Promise { // Call a function to create initial state for contracts. await this._baseIndexer.createInit(this, blockHash, blockNumber); + } + async processBlockAfterEvents (blockHash: string): Promise { // Call subgraph handler for block. await this._graphWatcher.handleBlock(blockHash); } @@ -749,10 +752,10 @@ export class Indexer implements IPLDIndexerInterface { txHash, contract, eventName, - eventInfo: JSONbig.stringify(eventInfo), - extraInfo: JSONbig.stringify(extraInfo), - proof: JSONbig.stringify({ - data: JSONbig.stringify({ + eventInfo: JSONbigNative.stringify(eventInfo), + extraInfo: JSONbigNative.stringify(extraInfo), + proof: JSONbigNative.stringify({ + data: JSONbigNative.stringify({ blockHash, receiptCID, log: { diff --git a/packages/lighthouse-watcher/src/indexer.ts b/packages/lighthouse-watcher/src/indexer.ts index 9d48ffce2..626e69302 100644 --- a/packages/lighthouse-watcher/src/indexer.ts +++ b/packages/lighthouse-watcher/src/indexer.ts @@ -15,6 +15,7 @@ import lighthouseABI from './abi/Lighthouse.json'; export const UNKNOWN_EVENT_NAME = '__unknown__'; const log = debug('vulcanize:indexer'); +const JSONbigNative = JSONbig({ useNativeBigInt: true }); export type ResultEvent = { block: any; @@ -160,7 +161,7 @@ export class Indexer { ...eventInfo }, proof: { - data: JSONbig.stringify({ + data: JSONbigNative.stringify({ blockHash, receiptCID, log: { diff --git a/packages/mobymask-watcher/src/indexer.ts b/packages/mobymask-watcher/src/indexer.ts index adb9cf47c..ce930fce9 100644 --- a/packages/mobymask-watcher/src/indexer.ts +++ b/packages/mobymask-watcher/src/indexer.ts @@ -47,6 +47,7 @@ import { _Owner } from './entity/_Owner'; import { MultiNonce } from './entity/MultiNonce'; const log = debug('vulcanize:indexer'); +const JSONbigNative = JSONbig({ useNativeBigInt: true }); export const KIND_PHISHERREGISTRY = 'PhisherRegistry'; @@ -139,8 +140,8 @@ export class Indexer implements IPLDIndexerInterface { getResultEvent (event: Event): ResultEvent { const block = event.block; - const eventFields = JSONbig.parse(event.eventInfo); - const { tx, eventSignature } = JSON.parse(event.extraInfo); + const eventFields = JSONbigNative.parse(event.eventInfo); + const { tx, eventSignature } = JSONbigNative.parse(event.extraInfo); return { block: { @@ -395,7 +396,7 @@ export class Indexer implements IPLDIndexerInterface { contractAddress, ...mappingKeys, value: result.value, - proof: result.proof ? JSONbig.stringify(result.proof) : null + proof: result.proof ? JSONbigNative.stringify(result.proof) : null } as any; } @@ -801,10 +802,10 @@ export class Indexer implements IPLDIndexerInterface { txHash, contract, eventName, - eventInfo: JSONbig.stringify(eventInfo), - extraInfo: JSONbig.stringify(extraInfo), - proof: JSONbig.stringify({ - data: JSONbig.stringify({ + eventInfo: JSONbigNative.stringify(eventInfo), + extraInfo: JSONbigNative.stringify(extraInfo), + proof: JSONbigNative.stringify({ + data: JSONbigNative.stringify({ blockHash, receiptCID, log: { diff --git a/packages/solidity-mapper/src/storage.test.ts b/packages/solidity-mapper/src/storage.test.ts index 800d175e5..59064c5a4 100644 --- a/packages/solidity-mapper/src/storage.test.ts +++ b/packages/solidity-mapper/src/storage.test.ts @@ -1073,10 +1073,10 @@ describe('Get value from storage', () => { before(async () => { ({ contract: testNestedArrays, storageLayout } = contracts.TestNestedArrays); - const transactions = []; + const transactions: Array = []; const addresses = generateDummyAddresses(7); - const transactionPromises = []; + const transactionPromises: Promise[] = []; // Set value for nestedStructArray. for (let i = 0; i < 5; i++) { diff --git a/packages/uni-info-watcher/src/indexer.ts b/packages/uni-info-watcher/src/indexer.ts index 23e90bd48..457199e70 100644 --- a/packages/uni-info-watcher/src/indexer.ts +++ b/packages/uni-info-watcher/src/indexer.ts @@ -36,6 +36,7 @@ import { Tick } from './entity/Tick'; const SYNC_DELTA = 5; const log = debug('vulcanize:indexer'); +const JSONbigNative = JSONbig({ useNativeBigInt: true }); export { OrderDirection, BlockHeight }; @@ -69,8 +70,8 @@ export class Indexer implements IndexerInterface { getResultEvent (event: Event): ResultEvent { const block = event.block; - const eventFields = JSON.parse(event.eventInfo); - const { tx, eventIndex } = JSON.parse(event.extraInfo); + const eventFields = JSONbigNative.parse(event.eventInfo); + const { tx, eventIndex } = JSONbigNative.parse(event.extraInfo); return { block: { @@ -394,9 +395,9 @@ export class Indexer implements IndexerInterface { txHash: tx.hash, contract, eventName, - eventInfo: JSONbig.stringify(eventInfo), - extraInfo: JSONbig.stringify(extraInfo), - proof: JSONbig.stringify(proof) + eventInfo: JSONbigNative.stringify(eventInfo), + extraInfo: JSONbigNative.stringify(extraInfo), + proof: JSONbigNative.stringify(proof) }); } diff --git a/packages/uni-watcher/src/indexer.ts b/packages/uni-watcher/src/indexer.ts index cc1222ed8..89ad6c8dd 100644 --- a/packages/uni-watcher/src/indexer.ts +++ b/packages/uni-watcher/src/indexer.ts @@ -22,6 +22,7 @@ import { abi as nfpmABI, storageLayout as nfpmStorageLayout } from './artifacts/ import poolABI from './artifacts/pool.json'; const log = debug('vulcanize:indexer'); +const JSONbigNative = JSONbig({ useNativeBigInt: true }); type ResultEvent = { block: any; @@ -68,8 +69,8 @@ export class Indexer implements IndexerInterface { getResultEvent (event: Event): ResultEvent { const block = event.block; - const eventFields = JSON.parse(event.eventInfo); - const { tx } = JSON.parse(event.extraInfo); + const eventFields = JSONbigNative.parse(event.eventInfo); + const { tx } = JSONbigNative.parse(event.extraInfo); return { block: { @@ -521,10 +522,10 @@ export class Indexer implements IndexerInterface { txHash, contract, eventName, - eventInfo: JSONbig.stringify(eventInfo), - extraInfo: JSONbig.stringify(extraInfo), - proof: JSONbig.stringify({ - data: JSONbig.stringify({ + eventInfo: JSONbigNative.stringify(eventInfo), + extraInfo: JSONbigNative.stringify(extraInfo), + proof: JSONbigNative.stringify({ + data: JSONbigNative.stringify({ blockHash, receiptCID, log: { diff --git a/packages/util/src/common.ts b/packages/util/src/common.ts index c8b135e61..796e158c8 100644 --- a/packages/util/src/common.ts +++ b/packages/util/src/common.ts @@ -191,4 +191,8 @@ export const processBatchEvents = async (indexer: IndexerInterface, block: Block console.timeEnd('time:common#processBacthEvents-processing_events_batch'); } + + if (indexer.processBlockAfterEvents) { + await indexer.processBlockAfterEvents(block.blockHash); + } }; diff --git a/packages/util/src/types.ts b/packages/util/src/types.ts index 224814ab1..d9b55dc32 100644 --- a/packages/util/src/types.ts +++ b/packages/util/src/types.ts @@ -107,6 +107,7 @@ export interface IndexerInterface { processInitialState?: (contractAddress: string, blockHash: string) => Promise processStateCheckpoint?: (contractAddress: string, blockHash: string) => Promise processBlock?: (blockHash: string, blockNumber: number) => Promise + processBlockAfterEvents?: (blockHash: string) => Promise } export interface IPLDIndexerInterface extends IndexerInterface { diff --git a/yarn.lock b/yarn.lock index 6df36bdc5..5add9a0c4 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2581,6 +2581,11 @@ dependencies: "@types/node" "*" +"@types/pluralize@^0.0.29": + version "0.0.29" + resolved "https://registry.yarnpkg.com/@types/pluralize/-/pluralize-0.0.29.tgz#6ffa33ed1fc8813c469b859681d09707eb40d03c" + integrity sha512-BYOID+l2Aco2nBik+iYS4SZX0Lf20KPILP5RGmM1IgzdwNdTs0eebiFriOPcej1sX9mLnSoiNte5zcFxssgpGA== + "@types/prettier@^2.1.1": version "2.2.3" resolved "https://registry.yarnpkg.com/@types/prettier/-/prettier-2.2.3.tgz#ef65165aea2924c9359205bf748865b8881753c0" @@ -11561,6 +11566,11 @@ please-upgrade-node@^3.2.0: dependencies: semver-compare "^1.0.0" +pluralize@^8.0.0: + version "8.0.0" + resolved "https://registry.yarnpkg.com/pluralize/-/pluralize-8.0.0.tgz#1a6fa16a38d12a1901e0320fa017051c539ce3b1" + integrity sha512-Nc3IT5yHzflTfbjgqWcCPpo7DaKy4FnpB0l/zCAW0Tc7jxAiuqSxHasntB3D7887LSrA93kDJ9IXovxJYxyLCA== + posix-character-classes@^0.1.0: version "0.1.1" resolved "https://registry.yarnpkg.com/posix-character-classes/-/posix-character-classes-0.1.1.tgz#01eac0fe3b5af71a2a6c02feabb8c1fef7e00eab"