-
Notifications
You must be signed in to change notification settings - Fork 3
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add a CLI to backfill watcher event data (#538)
* Add a CLI to backfill watcher event data * Add required codegen template * Increment package version
- Loading branch information
1 parent
5d7b7fe
commit ac74da6
Showing
18 changed files
with
216 additions
and
35 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,127 @@ | ||
| // | ||
| // Copyright 2024 Vulcanize, Inc. | ||
| // | ||
|
|
||
| import yargs from 'yargs'; | ||
| import { hideBin } from 'yargs/helpers'; | ||
| import assert from 'assert'; | ||
| import { ConnectionOptions, Repository } from 'typeorm'; | ||
| import debug from 'debug'; | ||
|
|
||
| import { DEFAULT_CONFIG_PATH, JSONbigNative, DatabaseInterface, Config, EventInterface } from '@cerc-io/util'; | ||
|
|
||
| import { BaseCmd } from './base'; | ||
|
|
||
| const log = debug('vulcanize:backfill-events-data'); | ||
|
|
||
| interface Arguments { | ||
| configFile: string; | ||
| batchSize: number; | ||
| } | ||
|
|
||
| export class BackfillEventsDataCmd { | ||
| _argv?: Arguments; | ||
| _baseCmd: BaseCmd; | ||
|
|
||
| constructor () { | ||
| this._baseCmd = new BaseCmd(); | ||
| } | ||
|
|
||
| get config (): Config { | ||
| return this._baseCmd.config; | ||
| } | ||
|
|
||
| get database (): DatabaseInterface { | ||
| return this._baseCmd.database; | ||
| } | ||
|
|
||
| async initConfig<ConfigType> (): Promise<ConfigType> { | ||
| this._argv = this._getArgv(); | ||
| assert(this._argv); | ||
|
|
||
| return this._baseCmd.initConfig(this._argv.configFile); | ||
| } | ||
|
|
||
| async init ( | ||
| Database: new ( | ||
| config: ConnectionOptions | ||
| ) => DatabaseInterface | ||
| ): Promise<void> { | ||
| await this.initConfig(); | ||
|
|
||
| this._baseCmd._database = new Database(this.config.database); | ||
| await this.database.init(); | ||
| } | ||
|
|
||
| async exec (eventEntity: new () => EventInterface): Promise<void> { | ||
| assert(this._argv); | ||
|
|
||
| const eventRepository: Repository<EventInterface> = this.database._conn.getRepository(eventEntity); | ||
|
|
||
| // Get the total count of events | ||
| const totalEvents = await eventRepository.count(); | ||
|
|
||
| const batchSize = Number(this._argv.batchSize); | ||
| let page = 0; | ||
| let processedCount = 0; | ||
| let eventsWithNullData: EventInterface[]; | ||
|
|
||
| while (processedCount < totalEvents) { | ||
| // Fetch events in batches with pagination | ||
| eventsWithNullData = await eventRepository.find({ | ||
| order: { id: 'ASC' }, | ||
| skip: page * batchSize, | ||
| take: batchSize | ||
| }); | ||
|
|
||
| for (const event of eventsWithNullData) { | ||
| // Parse extra info and check if data field is present | ||
| const parsedExtraInfo = JSON.parse(event.extraInfo); | ||
|
|
||
| // Derive data and topics | ||
| if (parsedExtraInfo.data) { | ||
| event.data = parsedExtraInfo.data; | ||
| [event.topic0, event.topic1, event.topic2, event.topic3] = parsedExtraInfo.topics; | ||
|
|
||
| // Update extraInfo | ||
| delete parsedExtraInfo.data; | ||
| delete parsedExtraInfo.topics; | ||
|
|
||
| event.extraInfo = JSONbigNative.stringify(parsedExtraInfo); | ||
| } | ||
| } | ||
|
|
||
| // Save updated events | ||
| await eventRepository.save(eventsWithNullData); | ||
|
|
||
| // Update the processed count and progress | ||
| processedCount += eventsWithNullData.length; | ||
| const progress = ((processedCount / totalEvents) * 100).toFixed(2); | ||
| log(`Processed ${processedCount}/${totalEvents} events (${progress}% complete)`); | ||
|
|
||
| // Move to the next batch | ||
| eventsWithNullData = []; | ||
| page++; | ||
| } | ||
|
|
||
| log('Done.'); | ||
| await this.database.close(); | ||
| } | ||
|
|
||
| _getArgv (): any { | ||
| return yargs(hideBin(process.argv)) | ||
| .option('configFile', { | ||
| alias: 'f', | ||
| describe: 'configuration file path (toml)', | ||
| type: 'string', | ||
| default: DEFAULT_CONFIG_PATH | ||
| }) | ||
| .option('b', { | ||
| alias: 'batch-size', | ||
| describe: 'batch size to process events in', | ||
| type: 'number', | ||
| default: 1000 | ||
| }) | ||
| .argv; | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,21 @@ | ||
| // | ||
| // Copyright 2024 Vulcanize, Inc. | ||
| // | ||
|
|
||
| import fs from 'fs'; | ||
| import path from 'path'; | ||
| import Handlebars from 'handlebars'; | ||
| import { Writable } from 'stream'; | ||
|
|
||
| const TEMPLATE_FILE = './templates/backfill-events-data-template.handlebars'; | ||
|
|
||
| /** | ||
| * Writes the backfill-events-data file generated from a template to a stream. | ||
| * @param outStream A writable output stream to write the backfill-events-data file to. | ||
| */ | ||
| export function exportBackfillEventsData (outStream: Writable): void { | ||
| const templateString = fs.readFileSync(path.resolve(__dirname, TEMPLATE_FILE)).toString(); | ||
| const template = Handlebars.compile(templateString); | ||
| const content = template({}); | ||
| outStream.write(content); | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
26 changes: 26 additions & 0 deletions
26
packages/codegen/src/templates/backfill-events-data-template.handlebars
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,26 @@ | ||
| // | ||
| // Copyright 2024 Vulcanize, Inc. | ||
| // | ||
|
|
||
| import 'reflect-metadata'; | ||
| import debug from 'debug'; | ||
|
|
||
| import { BackfillEventsDataCmd } from '@cerc-io/cli'; | ||
|
|
||
| import { Database } from '../database'; | ||
| import { Event } from '../entity/Event'; | ||
|
|
||
| const log = debug('vulcanize:backfill-events-data'); | ||
|
|
||
| const main = async (): Promise<void> => { | ||
| const backFillCmd = new BackfillEventsDataCmd(); | ||
| await backFillCmd.init(Database); | ||
|
|
||
| await backFillCmd.exec(Event); | ||
| }; | ||
|
|
||
| main().catch(err => { | ||
| log(err); | ||
| }).finally(() => { | ||
| process.exit(0); | ||
| }); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.