From cf8ec6e14e384092f3f91fd5b880ac2e1f1d4750 Mon Sep 17 00:00:00 2001 From: amateima <89395931+amateima@users.noreply.github.com> Date: Fri, 13 Jan 2023 23:52:52 +0200 Subject: [PATCH] feat: split modes dynamic modules (prod) (#207) --- .env.sample | 48 +++-- .github/workflows/lint-build-test.yaml | 1 + README.md | 51 +++++ src/app.module.ts | 77 ++++---- src/dynamic-module.ts | 9 + src/main.ts | 5 +- .../adapter/db/claim-fixture.ts | 0 .../airdrop/entry-points/http/controller.ts | 2 + .../model/claim.entity.ts | 2 +- .../model/merkle-distributor-window.entity.ts | 2 +- src/modules/airdrop/module.ts | 73 ++++--- .../airdrop/services/airdrop-service.ts | 4 +- src/modules/auth/auth.module.ts | 54 ++++-- .../entry-points/http/discord.strategy.ts | 3 +- src/modules/configuration/index.ts | 2 + src/modules/database/database.providers.ts | 4 +- .../adapter/db/deposit-fixture.ts | 0 .../model/deposit.entity.ts | 0 src/modules/deposit/module.ts | 38 +++- src/modules/deposit/service.ts | 2 +- src/modules/health/health.controller.ts | 2 +- src/modules/market-price/module.ts | 35 +++- src/modules/referral/module.ts | 39 +++- src/modules/referral/services/cron-service.ts | 2 +- src/modules/referral/services/service.ts | 4 +- .../adapter/messaging/BlockNumberConsumer.ts | 2 +- .../adapter/messaging/BlocksEventsConsumer.ts | 2 +- .../messaging/DepositAcxPriceConsumer.ts | 2 +- .../messaging/DepositFilledDateConsumer.ts | 2 +- .../messaging/DepositReferralConsumer.ts | 2 +- .../adapter/messaging/FillEventsConsumer.ts | 2 +- .../MerkleDistributorBlocksEventsConsumer.ts | 2 +- .../messaging/SpeedUpEventsConsumer.ts | 2 +- .../messaging/SuggestedFeesConsumer.ts | 2 +- .../adapter/messaging/TokenDetailsConsumer.ts | 2 +- .../adapter/messaging/TokenPriceConsumer.ts | 2 +- .../messaging/TrackFillEventConsumer.ts | 2 +- src/modules/scraper/module.ts | 180 +++++++++--------- .../scraper/service/ScraperQueuesService.ts | 5 +- src/modules/user/module.ts | 35 +++- test/airdrop.e2e-spec.ts | 7 +- test/airdrop/merkle-distributor.e2e-spec.ts | 3 +- test/deposit.e2e-spec.ts | 5 +- test/referrals.e2e-spec.ts | 7 +- test/run-modes.e2e-spec.ts | 30 +++ test/scraper.e2e-spec.ts | 5 +- test/user.e2e-spec.ts | 11 +- test/utils.ts | 7 - 48 files changed, 517 insertions(+), 261 deletions(-) create mode 100644 src/dynamic-module.ts rename src/modules/{scraper => airdrop}/adapter/db/claim-fixture.ts (100%) rename src/modules/{scraper => airdrop}/model/claim.entity.ts (89%) rename src/modules/{scraper => deposit}/adapter/db/deposit-fixture.ts (100%) rename src/modules/{scraper => deposit}/model/deposit.entity.ts (100%) create mode 100644 test/run-modes.e2e-spec.ts delete mode 100644 test/utils.ts diff --git a/.env.sample b/.env.sample index e0bc4b4d..6723fc0f 100644 --- a/.env.sample +++ b/.env.sample @@ -1,3 +1,6 @@ +######################## +# SHARED ENV VARIABLES # +######################## PORT=3001 DB_HOST=postgres DB_PORT=5432 @@ -7,33 +10,46 @@ DB_DATABASE_NAME=scraperapi REDIS_HOST=redis REDIS_PORT=6379 REDIS_PASSWORD=password +JWT_SECRET=secret +# MerkleDistributor overrides +MERKLE_DISTRIBUTOR_CHAIN_ID= +MERKLE_DISTRIBUTOR_ADDRESS= +# set the list of operation modes ("normal", "test", "scraper") +RUN_MODES=normal + +######################## +# NORMAL ENV VARIABLES # +######################## +DISTRIBUTOR_PROOFS_CACHE_SECONDS_DURATION= +REFERRALS_SUMMARY_CACHE_SECONDS_DURATION= +# the Discord app's credentials obtained from https://discord.com/developers/applications. +DISCORD_CLIENT_ID=clientId +DISCORD_CLIENT_SECRET=clientSecret +# the url accessed after the Discord authorization processed is fulfilled +DISCORD_REDIRECT_URI=http://localhost +AMPLITUDE_API_KEY= + +######################### +# SCRAPER ENV VARIABLES # +######################### + WEB3_NODE_URL_1=https://mainnet.infura.io/v3/ WEB3_NODE_URL_10=https://optimism-mainnet.infura.io/v3/ WEB3_NODE_URL_288=https://boba-mainnet.gateway.pokt.network/v1/lb/ WEB3_NODE_URL_42161=https://arbitrum-mainnet.infura.io/v3/ WEB3_NODE_URL_137=https://polygon-mainnet.infura.io/v3/ WEB3_NODE_URL_5=https://goerli.infura.io/v3/ +# the timestamp after which the referral address is extracted using the delimiter REFERRAL_DELIMITER_START_TIMESTAMP=1657290720 +# enable fetching SpokePool events from the contracts ENABLE_SPOKE_POOLS_EVENTS_PROCESSING=false +# enable fetching MerkleDistributor events from the contracts +ENABLE_MERKLE_DISTRIBUTOR_EVENTS_PROCESSING=false +# enable the refresh of the referrals materialized view. ENABLE_REFERRALS_MATERIALIZED_VIEW_REFRESH=false # specify the strategy used for updating sticky referrals. Valid options: queue | cron | disable STICKY_REFERRAL_ADDRESSES_MECHANISM=queue -DISABLE_CRONS=true # Following distances in blocks to guarantee finality on each chain. Format should be a map chainId -> blocks. # E.g. { "137": 100 } FOLLOWING_DISTANCES= -JWT_SECRET=secret -# DISCORD_CLIENT_ID and DISCORD_CLIENT_SECRET are the credentials of the Discord app and they are obtained from https://discord.com/developers/applications. -DISCORD_CLIENT_ID=clientId -DISCORD_CLIENT_SECRET=clientSecret -# the url accessed after the Discord authorization processed is fulfilled -DISCORD_REDIRECT_URI=http://localhost - -# MerkleDistributor overrides -MERKLE_DISTRIBUTOR_CHAIN_ID= -MERKLE_DISTRIBUTOR_ADDRESS= -ENABLE_MERKLE_DISTRIBUTOR_EVENTS_PROCESSING=false - -DISTRIBUTOR_PROOFS_CACHE_SECONDS_DURATION=0 -REFERRALS_SUMMARY_CACHE_SECONDS_DURATION=0 -AMPLITUDE_API_KEY= +DISABLE_CRONS=true diff --git a/.github/workflows/lint-build-test.yaml b/.github/workflows/lint-build-test.yaml index 45169842..710cf5fd 100644 --- a/.github/workflows/lint-build-test.yaml +++ b/.github/workflows/lint-build-test.yaml @@ -43,6 +43,7 @@ jobs: DISCORD_REDIRECT_URI: http://localhost DISTRIBUTOR_PROOFS_CACHE_SECONDS_DURATION: 0 REFERRALS_SUMMARY_CACHE_SECONDS_DURATION: 0 + RUN_MODES: normal,test,scraper steps: - uses: actions/checkout@v3 - uses: ./.github/actions/setup diff --git a/README.md b/README.md index aa1f1b07..eac10ae7 100644 --- a/README.md +++ b/README.md @@ -36,3 +36,54 @@ git rebase master - `feature-branch` -> `master`: The branch will be deployed on the production environment for testing purposes. Most of the time, all feature branches will be merged into `stage` branch before the `master` branch. 3. Update the `stage` branch with the latest master whenever is needed + +## Run modes + +The application has the ability the load modules and dependencies depending on predefined running modes that can be one or multiple of: scraper, normal, test. Use the RUN_MODES env variable to configure the behaviour of the application: + +``` +RUN_MODES=normal,test +``` + +In order to configure a module to behave differently depending on the running mode, use the static `forRoot` method like in the example below: + +```ts +@Module({}) +export class ExampleModule { + static forRoot(moduleOptions: ModuleOptions): DynamicModule { + let module: DynamicModule = { module: ExampleModule, providers: [], controllers: [], imports: [], exports: [] }; + + if (moduleOptions.runModes.includes(RunMode.Normal)) { + module = { + ...module, + controllers: [...module.controllers, ...], + providers: [...module.providers, ...], + imports: [...module.imports, ...], + exports: [...module.exports, ...], + }; + } + + if (moduleOptions.runModes.includes(RunMode.Scraper)) { + module = { + ...module, + controllers: [...module.controllers, ...], + providers: [...module.providers, ...], + imports: [...module.imports, ...], + exports: [...module.exports, ...], + }; + } + + if (moduleOptions.runModes.includes(RunMode.Test)) { + module = { + ...module, + controllers: [...module.controllers, ...], + providers: [...module.providers, ...], + imports: [...module.imports, ...], + exports: [...module.exports, ...], + }; + } + + return module; + } +} +``` \ No newline at end of file diff --git a/src/app.module.ts b/src/app.module.ts index 98398f21..ffb6dcb9 100644 --- a/src/app.module.ts +++ b/src/app.module.ts @@ -1,10 +1,10 @@ import { BullModule } from "@nestjs/bull"; -import { CacheModule, MiddlewareConsumer, Module } from "@nestjs/common"; +import { CacheModule, MiddlewareConsumer, Module, DynamicModule } from "@nestjs/common"; import { ConfigModule } from "@nestjs/config"; import { ScheduleModule } from "@nestjs/schedule"; import { TypeOrmModule } from "@nestjs/typeorm"; import { LogsMiddleware } from "./logging.interceptor"; -import configuration from "./modules/configuration"; +import configuration, { configValues } from "./modules/configuration"; import { DatabaseModule } from "./modules/database/database.module"; import { TypeOrmDefaultConfigService } from "./modules/database/database.providers"; import { HealthModule } from "./modules/health/health.module"; @@ -18,38 +18,51 @@ import { DepositModule } from "./modules/deposit/module"; import { AuthModule } from "./modules/auth/auth.module"; import { UserModule } from "./modules/user/module"; import { AirdropModule } from "./modules/airdrop/module"; +import { ModuleOptions, RunMode } from "./dynamic-module"; -@Module({ - imports: [ - ConfigModule.forRoot({ - ignoreEnvFile: false, - ignoreEnvVars: false, - isGlobal: true, - expandVariables: true, - load: [configuration], - }), - TypeOrmModule.forRootAsync({ - imports: [DatabaseModule], - useExisting: TypeOrmDefaultConfigService, - }), - HealthModule, - Web3Module, - ScraperModule, - BullModule.forRootAsync({ - imports: [MessagingModule], - useExisting: BullConfigService, - }), - ReferralModule, - MarketPriceModule, - ScheduleModule.forRoot(), - DepositModule, - AuthModule, - UserModule, - AirdropModule, - CacheModule.register({ isGlobal: true }), - ], -}) +@Module({}) export class AppModule { + static forRoot(moduleOptions: ModuleOptions): DynamicModule { + const imports = [ + ConfigModule.forRoot({ + ignoreEnvFile: false, + ignoreEnvVars: false, + isGlobal: true, + expandVariables: true, + load: [configuration], + }), + TypeOrmModule.forRootAsync({ + imports: [DatabaseModule], + useExisting: TypeOrmDefaultConfigService, + }), + HealthModule, + Web3Module, + ReferralModule.forRoot(moduleOptions), + MarketPriceModule.forRoot(moduleOptions), + ScheduleModule.forRoot(), + DepositModule.forRoot(moduleOptions), + AuthModule.forRoot(moduleOptions), + UserModule.forRoot(moduleOptions), + AirdropModule.forRoot(moduleOptions), + CacheModule.register({ isGlobal: true }), + ]; + + if (moduleOptions.runModes.includes(RunMode.Scraper)) { + imports.push( + ScraperModule.forRoot(moduleOptions), + BullModule.forRootAsync({ + imports: [MessagingModule], + useExisting: BullConfigService, + }), + ); + } + + return { + module: AppModule, + imports, + }; + } + configure(consumer: MiddlewareConsumer) { consumer.apply(LogsMiddleware).forRoutes("*"); } diff --git a/src/dynamic-module.ts b/src/dynamic-module.ts new file mode 100644 index 00000000..2328d4a8 --- /dev/null +++ b/src/dynamic-module.ts @@ -0,0 +1,9 @@ +export enum RunMode { + Scraper = "scraper", + Normal = "normal", + Test = "test", +} + +export interface ModuleOptions { + runModes: RunMode[]; +} diff --git a/src/main.ts b/src/main.ts index 857978a0..044ff7b5 100644 --- a/src/main.ts +++ b/src/main.ts @@ -6,9 +6,12 @@ import { AppModule } from "./app.module"; import { AppConfig } from "./modules/configuration/configuration.service"; import { DocumentBuilder, SwaggerModule } from "@nestjs/swagger"; import { ValidationPipe } from "./validation.pipe"; +import { configValues } from "./modules/configuration"; async function bootstrap() { - const app = await NestFactory.create(AppModule); + const app = await NestFactory.create( + AppModule.forRoot({ runModes: configValues().app.runModes }), + ); const config = app.get(AppConfig); const port = config.values.app.port; diff --git a/src/modules/scraper/adapter/db/claim-fixture.ts b/src/modules/airdrop/adapter/db/claim-fixture.ts similarity index 100% rename from src/modules/scraper/adapter/db/claim-fixture.ts rename to src/modules/airdrop/adapter/db/claim-fixture.ts diff --git a/src/modules/airdrop/entry-points/http/controller.ts b/src/modules/airdrop/entry-points/http/controller.ts index d7849aa5..4447ae7f 100644 --- a/src/modules/airdrop/entry-points/http/controller.ts +++ b/src/modules/airdrop/entry-points/http/controller.ts @@ -77,6 +77,7 @@ export class AirdropController { ), ) @ApiTags("airdrop") + @ApiBearerAuth() async feedWalletRewards( @UploadedFiles() files: { walletRewards?: Express.Multer.File[]; communityRewards?: Express.Multer.File[] }, ) { @@ -96,6 +97,7 @@ export class AirdropController { }), ) @ApiTags("airdrop") + @ApiBearerAuth() uploadMerkleDistributorRecipients(@UploadedFile() file: Express.Multer.File) { return this.airdropService.processMerkleDistributorRecipientsFile(file); } diff --git a/src/modules/scraper/model/claim.entity.ts b/src/modules/airdrop/model/claim.entity.ts similarity index 89% rename from src/modules/scraper/model/claim.entity.ts rename to src/modules/airdrop/model/claim.entity.ts index fccf3db7..6e19b0b4 100644 --- a/src/modules/scraper/model/claim.entity.ts +++ b/src/modules/airdrop/model/claim.entity.ts @@ -8,7 +8,7 @@ import { UpdateDateColumn, ManyToOne, } from "typeorm"; -import { MerkleDistributorWindow } from "../../airdrop/model/merkle-distributor-window.entity"; +import { MerkleDistributorWindow } from "./merkle-distributor-window.entity"; @Entity() @Unique("UK_claim_windowIndex_accountIndex", ["windowIndex", "accountIndex"]) diff --git a/src/modules/airdrop/model/merkle-distributor-window.entity.ts b/src/modules/airdrop/model/merkle-distributor-window.entity.ts index 8a4364ca..b1f78e3c 100644 --- a/src/modules/airdrop/model/merkle-distributor-window.entity.ts +++ b/src/modules/airdrop/model/merkle-distributor-window.entity.ts @@ -1,6 +1,6 @@ import { Column, CreateDateColumn, Entity, OneToMany, PrimaryGeneratedColumn, Unique } from "typeorm"; import { MerkleDistributorRecipient } from "./merkle-distributor-recipient.entity"; -import { Claim } from "../../scraper/model/claim.entity"; +import { Claim } from "./claim.entity"; @Entity() // Don't allow duplicates of the window index diff --git a/src/modules/airdrop/module.ts b/src/modules/airdrop/module.ts index 49ef1656..6434fe84 100644 --- a/src/modules/airdrop/module.ts +++ b/src/modules/airdrop/module.ts @@ -1,4 +1,4 @@ -import { Module } from "@nestjs/common"; +import { Module, Provider, DynamicModule } from "@nestjs/common"; import { TypeOrmModule } from "@nestjs/typeorm"; import { UserModule } from "../user/module"; @@ -11,32 +11,55 @@ import { AirdropController } from "./entry-points/http/controller"; import { CommunityRewards } from "./model/community-rewards.entity"; import { WalletRewards } from "./model/wallet-rewards.entity"; -import { Deposit } from "../scraper/model/deposit.entity"; +import { Deposit } from "../deposit/model/deposit.entity"; import { MerkleDistributorWindowFixture } from "./adapter/db/merkle-distributor-window-fixture"; import { MerkleDistributorWindow } from "./model/merkle-distributor-window.entity"; import { MerkleDistributorRecipient } from "./model/merkle-distributor-recipient.entity"; import { MerkleDistributorRecipientFixture } from "./adapter/db/merkle-distributor-recipient"; +import { ModuleOptions, RunMode } from "../../dynamic-module"; +import { ClaimFixture } from "./adapter/db/claim-fixture"; +import { Claim } from "./model/claim.entity"; -@Module({ - providers: [ - AirdropService, - WalletRewardsFixture, - CommunityRewardsFixture, - MerkleDistributorWindowFixture, - MerkleDistributorRecipientFixture, - ], - controllers: [AirdropController], - imports: [ - TypeOrmModule.forFeature([ - CommunityRewards, - WalletRewards, - Deposit, - MerkleDistributorWindow, - MerkleDistributorRecipient, - ]), - UserModule, - AppConfigModule, - ], - exports: [], -}) -export class AirdropModule {} +@Module({}) +export class AirdropModule { + static forRoot(moduleOptions: ModuleOptions): DynamicModule { + let module: DynamicModule = { module: AirdropModule, providers: [], controllers: [], imports: [], exports: [] }; + + if (moduleOptions.runModes.includes(RunMode.Normal) || moduleOptions.runModes.includes(RunMode.Test)) { + module = { + ...module, + providers: [...module.providers, AirdropService, ClaimFixture], + controllers: [...module.controllers, AirdropController], + imports: [ + ...module.imports, + TypeOrmModule.forFeature([ + CommunityRewards, + WalletRewards, + Deposit, + MerkleDistributorWindow, + MerkleDistributorRecipient, + Claim, + ]), + UserModule.forRoot(moduleOptions), + AppConfigModule, + ], + }; + } + + if (moduleOptions.runModes.includes(RunMode.Test)) { + module = { + ...module, + providers: [ + ...module.providers, + ClaimFixture, + WalletRewardsFixture, + CommunityRewardsFixture, + MerkleDistributorWindowFixture, + MerkleDistributorRecipientFixture, + ], + }; + } + + return module; + } +} diff --git a/src/modules/airdrop/services/airdrop-service.ts b/src/modules/airdrop/services/airdrop-service.ts index 1388c304..8c7052bd 100644 --- a/src/modules/airdrop/services/airdrop-service.ts +++ b/src/modules/airdrop/services/airdrop-service.ts @@ -6,7 +6,7 @@ import BigNumber from "bignumber.js"; import { DataSource, IsNull, Not, QueryFailedError, Repository } from "typeorm"; import { Cache } from "cache-manager"; -import { Deposit } from "../../scraper/model/deposit.entity"; +import { Deposit } from "../../deposit/model/deposit.entity"; import { CommunityRewards } from "../model/community-rewards.entity"; import { WalletRewards } from "../model/wallet-rewards.entity"; import { Token } from "../../web3/model/token.entity"; @@ -36,8 +36,6 @@ export class AirdropService { @InjectRepository(CommunityRewards) private communityRewardsRepository: Repository, @InjectRepository(WalletRewards) private walletRewardsRepository: Repository, @InjectRepository(Deposit) private depositRepository: Repository, - @InjectRepository(Deposit) private merkleDistributorWindowRepository: Repository, - @InjectRepository(Deposit) private merkleDistributorRecipientRepository: Repository, private userService: UserService, private appConfig: AppConfig, private dataSource: DataSource, diff --git a/src/modules/auth/auth.module.ts b/src/modules/auth/auth.module.ts index bf6b344e..1efddebd 100644 --- a/src/modules/auth/auth.module.ts +++ b/src/modules/auth/auth.module.ts @@ -1,5 +1,5 @@ import { HttpModule } from "@nestjs/axios"; -import { Module } from "@nestjs/common"; +import { Module, DynamicModule } from "@nestjs/common"; import { JwtModule } from "@nestjs/jwt"; import { PassportModule } from "@nestjs/passport"; import { UserModule } from "../user/module"; @@ -10,20 +10,40 @@ import { configValues } from "../configuration"; import { AppConfigModule } from "../configuration/configuration.module"; import { JwtStrategy } from "./entry-points/http/jwt.strategy"; import { DiscordApiService } from "./adapters/discord"; +import { ModuleOptions, RunMode } from "../../dynamic-module"; -@Module({ - providers: [JwtStrategy, DiscordStrategy, AuthService, DiscordApiService], - controllers: [AuthController], - imports: [ - PassportModule.register({}), - HttpModule, - UserModule, - JwtModule.register({ - secret: configValues().auth.jwtSecret, - signOptions: { expiresIn: "31d" }, - }), - AppConfigModule, - ], - exports: [], -}) -export class AuthModule {} +@Module({}) +export class AuthModule { + static forRoot(moduleOptions: ModuleOptions): DynamicModule { + let module: DynamicModule = { module: AuthModule, controllers: [], imports: [], exports: [], providers: [] }; + + if (moduleOptions.runModes.includes(RunMode.Normal) || moduleOptions.runModes.includes(RunMode.Test)) { + module = { + ...module, + providers: [...module.providers, JwtStrategy, DiscordStrategy, AuthService, DiscordApiService], + controllers: [...module.controllers, AuthController], + imports: [ + ...module.imports, + PassportModule.register({}), + HttpModule, + UserModule.forRoot(moduleOptions), + JwtModule.register({ + secret: configValues().auth.jwtSecret, + signOptions: { expiresIn: "31d" }, + }), + AppConfigModule, + ], + }; + } + + if (moduleOptions.runModes.includes(RunMode.Scraper)) { + module = { + ...module, + providers: [...module.providers, JwtStrategy], + imports: [...module.imports, PassportModule.register({})], + }; + } + + return module; + } +} diff --git a/src/modules/auth/entry-points/http/discord.strategy.ts b/src/modules/auth/entry-points/http/discord.strategy.ts index a16c76ce..ed8ab2d9 100644 --- a/src/modules/auth/entry-points/http/discord.strategy.ts +++ b/src/modules/auth/entry-points/http/discord.strategy.ts @@ -1,7 +1,6 @@ import { PassportStrategy } from "@nestjs/passport"; import { Injectable } from "@nestjs/common"; import { Strategy } from "passport-oauth2"; -import { HttpService } from "@nestjs/axios"; import { stringify } from "querystring"; import { AppConfig } from "../../../configuration/configuration.service"; import { DiscordApiService } from "../../adapters/discord"; @@ -10,7 +9,7 @@ const ACROSS_DISCORD_GUILD_ID = "887426921892315137"; @Injectable() export class DiscordStrategy extends PassportStrategy(Strategy, "discord") { - constructor(private http: HttpService, private config: AppConfig, private discordApi: DiscordApiService) { + constructor(private config: AppConfig, private discordApi: DiscordApiService) { super({ authorizationURL: `https://discordapp.com/api/oauth2/authorize?${stringify({ client_id: config.values.discord.clientId, diff --git a/src/modules/configuration/index.ts b/src/modules/configuration/index.ts index 2473bcb5..ef7d3b21 100644 --- a/src/modules/configuration/index.ts +++ b/src/modules/configuration/index.ts @@ -1,5 +1,6 @@ import { registerAs } from "@nestjs/config"; import { ChainIds } from "../web3/model/ChainId"; +import { RunMode } from "../../dynamic-module"; export enum StickyReferralAddressesMechanism { Queue = "queue", @@ -34,6 +35,7 @@ export const configValues = () => ({ ? 120 : parseInt(process.env.REFERRALS_SUMMARY_CACHE_SECONDS_DURATION), }, + runModes: process.env.RUN_MODES ? (process.env.RUN_MODES.split(",") as RunMode[]) : ([RunMode.Normal] as RunMode[]), }, web3: { providers: { diff --git a/src/modules/database/database.providers.ts b/src/modules/database/database.providers.ts index b465599e..6e8c7451 100644 --- a/src/modules/database/database.providers.ts +++ b/src/modules/database/database.providers.ts @@ -3,9 +3,9 @@ import { Injectable } from "@nestjs/common"; import { AppConfig } from "../configuration/configuration.service"; import { ProcessedBlock } from "../scraper/model/ProcessedBlock.entity"; import { MerkleDistributorProcessedBlock } from "../scraper/model/MerkleDistributorProcessedBlock.entity"; -import { Claim } from "../scraper/model/claim.entity"; +import { Claim } from "../airdrop/model/claim.entity"; import { Block } from "../web3/model/block.entity"; -import { Deposit } from "../scraper/model/deposit.entity"; +import { Deposit } from "../deposit/model/deposit.entity"; import { Token } from "../web3/model/token.entity"; import { Transaction } from "../web3/model/transaction.entity"; import { HistoricMarketPrice } from "../market-price/model/historic-market-price.entity"; diff --git a/src/modules/scraper/adapter/db/deposit-fixture.ts b/src/modules/deposit/adapter/db/deposit-fixture.ts similarity index 100% rename from src/modules/scraper/adapter/db/deposit-fixture.ts rename to src/modules/deposit/adapter/db/deposit-fixture.ts diff --git a/src/modules/scraper/model/deposit.entity.ts b/src/modules/deposit/model/deposit.entity.ts similarity index 100% rename from src/modules/scraper/model/deposit.entity.ts rename to src/modules/deposit/model/deposit.entity.ts diff --git a/src/modules/deposit/module.ts b/src/modules/deposit/module.ts index 373014d7..c44474eb 100644 --- a/src/modules/deposit/module.ts +++ b/src/modules/deposit/module.ts @@ -1,14 +1,34 @@ -import { Module } from "@nestjs/common"; +import { DynamicModule, Module } from "@nestjs/common"; import { TypeOrmModule } from "@nestjs/typeorm"; +import { ModuleOptions, RunMode } from "../../dynamic-module"; import { AppConfigModule } from "../configuration/configuration.module"; -import { Deposit } from "../scraper/model/deposit.entity"; +import { Deposit } from "./model/deposit.entity"; import { DepositController } from "./controller"; import { DepositService } from "./service"; +import { DepositFixture } from "./adapter/db/deposit-fixture"; -@Module({ - controllers: [DepositController], - exports: [DepositService], - providers: [DepositService], - imports: [TypeOrmModule.forFeature([Deposit]), AppConfigModule], -}) -export class DepositModule {} +@Module({}) +export class DepositModule { + static forRoot(moduleOptions: ModuleOptions): DynamicModule { + let module: DynamicModule = { module: DepositModule, providers: [], controllers: [], imports: [], exports: [] }; + + if (moduleOptions.runModes.includes(RunMode.Normal) || moduleOptions.runModes.includes(RunMode.Test)) { + module = { + ...module, + controllers: [...module.controllers, DepositController], + providers: [...module.providers, DepositService], + exports: [...module.exports, DepositService], + imports: [...module.imports, TypeOrmModule.forFeature([Deposit]), AppConfigModule], + }; + } + + if (moduleOptions.runModes.includes(RunMode.Test)) { + module = { + ...module, + providers: [...module.providers, DepositFixture], + }; + } + + return module; + } +} diff --git a/src/modules/deposit/service.ts b/src/modules/deposit/service.ts index 8a9e66e0..b577c0ef 100644 --- a/src/modules/deposit/service.ts +++ b/src/modules/deposit/service.ts @@ -4,7 +4,7 @@ import { InjectRepository } from "@nestjs/typeorm"; import { Repository } from "typeorm"; import { utils } from "ethers"; import { Cache } from "cache-manager"; -import { Deposit } from "../scraper/model/deposit.entity"; +import { Deposit } from "./model/deposit.entity"; import { getAvgFillTimeQuery, getTotalDepositsQuery, getTotalVolumeQuery } from "./adapter/db/queries"; import { AppConfig } from "../configuration/configuration.service"; import { InvalidAddressException } from "./exceptions"; diff --git a/src/modules/health/health.controller.ts b/src/modules/health/health.controller.ts index ef87ed1b..27ab7456 100644 --- a/src/modules/health/health.controller.ts +++ b/src/modules/health/health.controller.ts @@ -3,7 +3,7 @@ import { ApiTags } from "@nestjs/swagger"; import { HealthDto, HealthParamsDto } from "./health.dto"; import { HealthService } from "./health.service"; -@Controller("api/health") +@Controller("health") @ApiTags("health") export class HealthController { constructor(private healthService: HealthService) {} diff --git a/src/modules/market-price/module.ts b/src/modules/market-price/module.ts index 9f1020e9..270b7e77 100644 --- a/src/modules/market-price/module.ts +++ b/src/modules/market-price/module.ts @@ -1,15 +1,34 @@ import { HttpModule } from "@nestjs/axios"; -import { Module } from "@nestjs/common"; +import { DynamicModule, Module } from "@nestjs/common"; import { TypeOrmModule } from "@nestjs/typeorm"; -import { Deposit } from "../scraper/model/deposit.entity"; +import { ModuleOptions, RunMode } from "../../dynamic-module"; +import { Deposit } from "../deposit/model/deposit.entity"; import { CoinGeckoService } from "./adapters/coingecko"; import { HistoricMarketPriceFixture } from "./adapters/hmp-fixture"; import { HistoricMarketPrice } from "./model/historic-market-price.entity"; import { MarketPriceService } from "./services/service"; -@Module({ - providers: [MarketPriceService, CoinGeckoService, HistoricMarketPriceFixture], - exports: [MarketPriceService], - imports: [HttpModule, TypeOrmModule.forFeature([HistoricMarketPrice, Deposit])], -}) -export class MarketPriceModule {} +@Module({}) +export class MarketPriceModule { + static forRoot(moduleOptions: ModuleOptions): DynamicModule { + let module: DynamicModule = { module: MarketPriceModule, providers: [], controllers: [], imports: [], exports: [] }; + + if (moduleOptions.runModes.includes(RunMode.Normal) || moduleOptions.runModes.includes(RunMode.Scraper)) { + module = { + ...module, + exports: [...module.exports, MarketPriceService], + providers: [...module.providers, MarketPriceService, CoinGeckoService], + imports: [...module.imports, HttpModule, TypeOrmModule.forFeature([HistoricMarketPrice, Deposit])], + }; + } + + if (moduleOptions.runModes.includes(RunMode.Test)) { + module = { + ...module, + providers: [...module.providers, HistoricMarketPriceFixture], + }; + } + + return module; + } +} diff --git a/src/modules/referral/module.ts b/src/modules/referral/module.ts index e81e6912..e6f3b7c0 100644 --- a/src/modules/referral/module.ts +++ b/src/modules/referral/module.ts @@ -1,16 +1,37 @@ -import { Module } from "@nestjs/common"; +import { DynamicModule, Module, Provider } from "@nestjs/common"; import { TypeOrmModule } from "@nestjs/typeorm"; import { AppConfigModule } from "../configuration/configuration.module"; import { DepositsMv } from "../deposit/model/DepositsMv.entity"; -import { Deposit } from "../scraper/model/deposit.entity"; +import { Deposit } from "../deposit/model/deposit.entity"; import { ReferralController } from "./entry-points/http/controller"; import { ReferralCronService } from "./services/cron-service"; import { ReferralService } from "./services/service"; +import { ModuleOptions, RunMode } from "../../dynamic-module"; -@Module({ - controllers: [ReferralController], - exports: [ReferralService], - providers: [ReferralService, ReferralCronService], - imports: [TypeOrmModule.forFeature([Deposit, DepositsMv]), AppConfigModule], -}) -export class ReferralModule {} +@Module({}) +export class ReferralModule { + static forRoot(moduleOptions: ModuleOptions): DynamicModule { + let module: DynamicModule = { module: ReferralModule, providers: [], controllers: [], imports: [], exports: [] }; + + if (moduleOptions.runModes.includes(RunMode.Normal)) { + module = { + ...module, + controllers: [...module.controllers, ReferralController], + providers: [...module.providers, ReferralService], + imports: [...module.imports, TypeOrmModule.forFeature([Deposit, DepositsMv]), AppConfigModule], + exports: [...module.exports, ReferralService], + }; + } + + if (moduleOptions.runModes.includes(RunMode.Scraper)) { + module = { + ...module, + providers: [...module.providers, ReferralCronService, ReferralService], + imports: [...module.imports, TypeOrmModule.forFeature([Deposit, DepositsMv]), AppConfigModule], + exports: [...module.exports, ReferralService], + }; + } + + return module; + } +} diff --git a/src/modules/referral/services/cron-service.ts b/src/modules/referral/services/cron-service.ts index 5049a5ac..e7542dea 100644 --- a/src/modules/referral/services/cron-service.ts +++ b/src/modules/referral/services/cron-service.ts @@ -2,7 +2,7 @@ import { Injectable, Logger } from "@nestjs/common"; import { Repository } from "typeorm"; import { InjectRepository } from "@nestjs/typeorm"; import { AppConfig } from "../../configuration/configuration.service"; -import { Deposit } from "../../scraper/model/deposit.entity"; +import { Deposit } from "../../deposit/model/deposit.entity"; import { EnhancedCron } from "../../../utils"; import { updateStickyReferralAddressesQuery } from "../services/queries"; import { StickyReferralAddressesMechanism } from "../../configuration"; diff --git a/src/modules/referral/services/service.ts b/src/modules/referral/services/service.ts index 3562e6b1..e052e50e 100644 --- a/src/modules/referral/services/service.ts +++ b/src/modules/referral/services/service.ts @@ -7,7 +7,7 @@ import Bluebird from "bluebird"; import { ethers } from "ethers"; import { Cache } from "cache-manager"; -import { Deposit } from "../../scraper/model/deposit.entity"; +import { Deposit } from "../../deposit/model/deposit.entity"; import { getActiveRefereesCountQuery, getReferralsQuery, @@ -24,7 +24,7 @@ import { WindowAlreadySetException } from "./exceptions"; import { DepositsFilteredReferrals } from "../model/DepositsFilteredReferrals.entity"; import { DepositReferralStat } from "../../deposit/model/deposit-referral-stat.entity"; import { splitArrayInChunks } from "../../../utils"; -import { Claim } from "../../scraper/model/claim.entity"; +import { Claim } from "../../airdrop/model/claim.entity"; const REFERRAL_ADDRESS_DELIMITER = "d00dfeeddeadbeef"; const getReferralsSummaryCacheKey = (address: string) => `referrals:summary:${address}`; diff --git a/src/modules/scraper/adapter/messaging/BlockNumberConsumer.ts b/src/modules/scraper/adapter/messaging/BlockNumberConsumer.ts index 9978198d..f36e92e7 100644 --- a/src/modules/scraper/adapter/messaging/BlockNumberConsumer.ts +++ b/src/modules/scraper/adapter/messaging/BlockNumberConsumer.ts @@ -11,7 +11,7 @@ import { TokenDetailsQueueMessage, SuggestedFeesQueueMessage, } from "."; -import { Deposit } from "../../model/deposit.entity"; +import { Deposit } from "../../../deposit/model/deposit.entity"; import { ScraperQueuesService } from "../../service/ScraperQueuesService"; @Processor(ScraperQueue.BlockNumber) diff --git a/src/modules/scraper/adapter/messaging/BlocksEventsConsumer.ts b/src/modules/scraper/adapter/messaging/BlocksEventsConsumer.ts index c144148f..ece7a937 100644 --- a/src/modules/scraper/adapter/messaging/BlocksEventsConsumer.ts +++ b/src/modules/scraper/adapter/messaging/BlocksEventsConsumer.ts @@ -17,7 +17,7 @@ import { FilledRelayEvent, RequestedSpeedUpDepositEvent, } from "@across-protocol/contracts-v2/dist/typechain/SpokePool"; -import { Deposit } from "../../model/deposit.entity"; +import { Deposit } from "../../../deposit/model/deposit.entity"; import { ScraperQueuesService } from "../../service/ScraperQueuesService"; @Processor(ScraperQueue.BlocksEvents) diff --git a/src/modules/scraper/adapter/messaging/DepositAcxPriceConsumer.ts b/src/modules/scraper/adapter/messaging/DepositAcxPriceConsumer.ts index c9c1f0a0..b8de31b3 100644 --- a/src/modules/scraper/adapter/messaging/DepositAcxPriceConsumer.ts +++ b/src/modules/scraper/adapter/messaging/DepositAcxPriceConsumer.ts @@ -5,7 +5,7 @@ import { InjectRepository } from "@nestjs/typeorm"; import { Repository } from "typeorm"; import { DateTime } from "luxon"; import { DepositAcxPriceQueueMessage, ScraperQueue } from "."; -import { Deposit } from "../../model/deposit.entity"; +import { Deposit } from "../../../deposit/model/deposit.entity"; import { MarketPriceService } from "../../../market-price/services/service"; @Processor(ScraperQueue.DepositAcxPrice) diff --git a/src/modules/scraper/adapter/messaging/DepositFilledDateConsumer.ts b/src/modules/scraper/adapter/messaging/DepositFilledDateConsumer.ts index fa9c974c..2f9ec49c 100644 --- a/src/modules/scraper/adapter/messaging/DepositFilledDateConsumer.ts +++ b/src/modules/scraper/adapter/messaging/DepositFilledDateConsumer.ts @@ -6,7 +6,7 @@ import { Repository } from "typeorm"; import { DateTime } from "luxon"; import { EthProvidersService } from "../../../web3/services/EthProvidersService"; import { DepositFilledDateQueueMessage, ScraperQueue } from "."; -import { Deposit, DepositFillTx } from "../../model/deposit.entity"; +import { Deposit, DepositFillTx } from "../../../deposit/model/deposit.entity"; @Processor(ScraperQueue.DepositFilledDate) export class DepositFilledDateConsumer { diff --git a/src/modules/scraper/adapter/messaging/DepositReferralConsumer.ts b/src/modules/scraper/adapter/messaging/DepositReferralConsumer.ts index 345a9239..9e18a698 100644 --- a/src/modules/scraper/adapter/messaging/DepositReferralConsumer.ts +++ b/src/modules/scraper/adapter/messaging/DepositReferralConsumer.ts @@ -4,7 +4,7 @@ import { Job } from "bull"; import { IsNull, Not, Repository } from "typeorm"; import { InjectRepository } from "@nestjs/typeorm"; import { DepositReferralQueueMessage, ScraperQueue } from "."; -import { Deposit } from "../../model/deposit.entity"; +import { Deposit } from "../../../deposit/model/deposit.entity"; import { EthProvidersService } from "../../../web3/services/EthProvidersService"; import { AppConfig } from "../../../configuration/configuration.service"; import { ReferralService } from "../../../referral/services/service"; diff --git a/src/modules/scraper/adapter/messaging/FillEventsConsumer.ts b/src/modules/scraper/adapter/messaging/FillEventsConsumer.ts index df5370af..e346a87f 100644 --- a/src/modules/scraper/adapter/messaging/FillEventsConsumer.ts +++ b/src/modules/scraper/adapter/messaging/FillEventsConsumer.ts @@ -9,7 +9,7 @@ import { TrackFillEventQueueMessage, } from "."; import { InjectRepository } from "@nestjs/typeorm"; -import { Deposit } from "../../model/deposit.entity"; +import { Deposit } from "../../../deposit/model/deposit.entity"; import { LessThan, MoreThan, Repository } from "typeorm"; import { BigNumber } from "bignumber.js"; import { ScraperQueuesService } from "../../service/ScraperQueuesService"; diff --git a/src/modules/scraper/adapter/messaging/MerkleDistributorBlocksEventsConsumer.ts b/src/modules/scraper/adapter/messaging/MerkleDistributorBlocksEventsConsumer.ts index ec6405ad..f6bb2d26 100644 --- a/src/modules/scraper/adapter/messaging/MerkleDistributorBlocksEventsConsumer.ts +++ b/src/modules/scraper/adapter/messaging/MerkleDistributorBlocksEventsConsumer.ts @@ -7,7 +7,7 @@ import { Repository, QueryFailedError } from "typeorm"; import { EthProvidersService } from "../../../web3/services/EthProvidersService"; import { MerkleDistributorBlocksEventsQueueMessage, ScraperQueue } from "."; import { ClaimedEvent } from "@across-protocol/contracts-v2/dist/typechain/MerkleDistributor"; -import { Claim } from "../../model/claim.entity"; +import { Claim } from "../../../airdrop/model/claim.entity"; import { utils } from "ethers"; @Processor(ScraperQueue.MerkleDistributorBlocksEvents) diff --git a/src/modules/scraper/adapter/messaging/SpeedUpEventsConsumer.ts b/src/modules/scraper/adapter/messaging/SpeedUpEventsConsumer.ts index 3ee48321..bba48f9c 100644 --- a/src/modules/scraper/adapter/messaging/SpeedUpEventsConsumer.ts +++ b/src/modules/scraper/adapter/messaging/SpeedUpEventsConsumer.ts @@ -3,7 +3,7 @@ import { Logger } from "@nestjs/common"; import { Job } from "bull"; import { DepositFilledDateQueueMessage, ScraperQueue, SpeedUpEventsQueueMessage } from "."; import { InjectRepository } from "@nestjs/typeorm"; -import { Deposit } from "../../model/deposit.entity"; +import { Deposit } from "../../../deposit/model/deposit.entity"; import { Repository } from "typeorm"; import { ScraperQueuesService } from "../../service/ScraperQueuesService"; diff --git a/src/modules/scraper/adapter/messaging/SuggestedFeesConsumer.ts b/src/modules/scraper/adapter/messaging/SuggestedFeesConsumer.ts index 432cf4e3..23df034b 100644 --- a/src/modules/scraper/adapter/messaging/SuggestedFeesConsumer.ts +++ b/src/modules/scraper/adapter/messaging/SuggestedFeesConsumer.ts @@ -8,7 +8,7 @@ import { utils } from "ethers"; import { SuggestedFeesService } from "../across-serverless-api/suggested-fees-service"; import { SuggestedFeesQueueMessage, ScraperQueue } from "."; -import { Deposit } from "../../model/deposit.entity"; +import { Deposit } from "../../../deposit/model/deposit.entity"; import { AppConfig } from "../../../configuration/configuration.service"; @Processor(ScraperQueue.SuggestedFees) diff --git a/src/modules/scraper/adapter/messaging/TokenDetailsConsumer.ts b/src/modules/scraper/adapter/messaging/TokenDetailsConsumer.ts index 13bcc804..06743bfa 100644 --- a/src/modules/scraper/adapter/messaging/TokenDetailsConsumer.ts +++ b/src/modules/scraper/adapter/messaging/TokenDetailsConsumer.ts @@ -3,7 +3,7 @@ import { Logger } from "@nestjs/common"; import { Job } from "bull"; import { ScraperQueue, TokenDetailsQueueMessage, TokenPriceQueueMessage } from "."; import { InjectRepository } from "@nestjs/typeorm"; -import { Deposit } from "../../model/deposit.entity"; +import { Deposit } from "../../../deposit/model/deposit.entity"; import { Repository } from "typeorm"; import { EthProvidersService } from "../../../web3/services/EthProvidersService"; import { ScraperQueuesService } from "../../service/ScraperQueuesService"; diff --git a/src/modules/scraper/adapter/messaging/TokenPriceConsumer.ts b/src/modules/scraper/adapter/messaging/TokenPriceConsumer.ts index e8304feb..e2821888 100644 --- a/src/modules/scraper/adapter/messaging/TokenPriceConsumer.ts +++ b/src/modules/scraper/adapter/messaging/TokenPriceConsumer.ts @@ -5,7 +5,7 @@ import { InjectRepository } from "@nestjs/typeorm"; import { Repository } from "typeorm"; import { DateTime } from "luxon"; import { DepositAcxPriceQueueMessage, ScraperQueue, TokenPriceQueueMessage } from "."; -import { Deposit } from "../../model/deposit.entity"; +import { Deposit } from "../../../deposit/model/deposit.entity"; import { MarketPriceService } from "../../../market-price/services/service"; import { ScraperQueuesService } from "../../service/ScraperQueuesService"; diff --git a/src/modules/scraper/adapter/messaging/TrackFillEventConsumer.ts b/src/modules/scraper/adapter/messaging/TrackFillEventConsumer.ts index 8df6a058..277dab14 100644 --- a/src/modules/scraper/adapter/messaging/TrackFillEventConsumer.ts +++ b/src/modules/scraper/adapter/messaging/TrackFillEventConsumer.ts @@ -8,7 +8,7 @@ import BigNumber from "bignumber.js"; import { DateTime } from "luxon"; import { TrackFillEventQueueMessage, ScraperQueue } from "."; -import { Deposit } from "../../model/deposit.entity"; +import { Deposit } from "../../../deposit/model/deposit.entity"; import { TrackService } from "../amplitude/track-service"; import { deriveRelayerFeeComponents, diff --git a/src/modules/scraper/module.ts b/src/modules/scraper/module.ts index 23614b4e..0b66ba11 100644 --- a/src/modules/scraper/module.ts +++ b/src/modules/scraper/module.ts @@ -1,14 +1,12 @@ import { HttpModule } from "@nestjs/axios"; import { BullModule } from "@nestjs/bull"; -import { Module } from "@nestjs/common"; +import { DynamicModule, Module, Provider } from "@nestjs/common"; import { TypeOrmModule } from "@nestjs/typeorm"; import { AppConfigModule } from "../configuration/configuration.module"; import { MarketPriceModule } from "../market-price/module"; import { ReferralModule } from "../referral/module"; import { Web3Module } from "../web3/module"; -import { DepositFixture } from "./adapter/db/deposit-fixture"; -import { ClaimFixture } from "./adapter/db/claim-fixture"; import { ScraperQueue } from "./adapter/messaging"; import { BlockNumberConsumer } from "./adapter/messaging/BlockNumberConsumer"; @@ -21,8 +19,8 @@ import { SpeedUpEventsConsumer } from "./adapter/messaging/SpeedUpEventsConsumer import { TokenDetailsConsumer } from "./adapter/messaging/TokenDetailsConsumer"; import { TokenPriceConsumer } from "./adapter/messaging/TokenPriceConsumer"; import { ScraperController } from "./entry-point/http/controller"; -import { Deposit } from "./model/deposit.entity"; -import { Claim } from "./model/claim.entity"; +import { Deposit } from "../deposit/model/deposit.entity"; +import { Claim } from "../airdrop/model/claim.entity"; import { ProcessedBlock } from "./model/ProcessedBlock.entity"; import { MerkleDistributorProcessedBlock } from "./model/MerkleDistributorProcessedBlock.entity"; import { ScraperService } from "./service"; @@ -32,85 +30,95 @@ import { SuggestedFeesConsumer } from "./adapter/messaging/SuggestedFeesConsumer import { SuggestedFeesService } from "./adapter/across-serverless-api/suggested-fees-service"; import { TrackFillEventConsumer } from "./adapter/messaging/TrackFillEventConsumer"; import { TrackService } from "./adapter/amplitude/track-service"; +import { ModuleOptions, RunMode } from "../../dynamic-module"; +import { DepositModule } from "../deposit/module"; +import { AirdropModule } from "../airdrop/module"; -@Module({ - providers: [ - ScraperService, - ScraperQueuesService, - SuggestedFeesService, - TrackService, - BlocksEventsConsumer, - MerkleDistributorBlocksEventsConsumer, - FillEventsConsumer, - SpeedUpEventsConsumer, - BlockNumberConsumer, - TokenDetailsConsumer, - DepositReferralConsumer, - TokenPriceConsumer, - DepositFilledDateConsumer, - DepositAcxPriceConsumer, - SuggestedFeesConsumer, - TrackFillEventConsumer, - DepositFixture, - ClaimFixture, - ], - imports: [ - Web3Module, - AppConfigModule, - TypeOrmModule.forFeature([ProcessedBlock, MerkleDistributorProcessedBlock, Claim, Deposit]), - MarketPriceModule, - HttpModule, - ReferralModule, - BullModule.registerQueue({ - name: ScraperQueue.BlockNumber, - }), - BullModule.registerQueue({ - name: ScraperQueue.BlocksEvents, - }), - BullModule.registerQueue({ - name: ScraperQueue.MerkleDistributorBlocksEvents, - }), - BullModule.registerQueue({ - name: ScraperQueue.TokenDetails, - }), - BullModule.registerQueue({ - name: ScraperQueue.TokenPrice, - }), - BullModule.registerQueue({ - name: ScraperQueue.DepositReferral, - }), - BullModule.registerQueue({ - name: ScraperQueue.DepositAcxPrice, - }), - BullModule.registerQueue({ - name: ScraperQueue.FillEvents, - defaultJobOptions: { - backoff: 120 * 1000, - attempts: Number.MAX_SAFE_INTEGER, - removeOnComplete: true, - removeOnFail: true, - }, - }), - BullModule.registerQueue({ - name: ScraperQueue.SpeedUpEvents, - defaultJobOptions: { - backoff: 120 * 1000, - attempts: Number.MAX_SAFE_INTEGER, - removeOnComplete: true, - removeOnFail: true, - }, - }), - BullModule.registerQueue({ - name: ScraperQueue.DepositFilledDate, - }), - BullModule.registerQueue({ - name: ScraperQueue.SuggestedFees, - }), - BullModule.registerQueue({ - name: ScraperQueue.TrackFillEvent, - }), - ], - exports: [ScraperQueuesService], - controllers: [ScraperController], -}) -export class ScraperModule {} +@Module({}) +export class ScraperModule { + static forRoot(moduleOptions: ModuleOptions): DynamicModule { + const providers: Provider[] = [ + ScraperService, + ScraperQueuesService, + SuggestedFeesService, + TrackService, + BlocksEventsConsumer, + MerkleDistributorBlocksEventsConsumer, + FillEventsConsumer, + SpeedUpEventsConsumer, + BlockNumberConsumer, + TokenDetailsConsumer, + DepositReferralConsumer, + TokenPriceConsumer, + DepositFilledDateConsumer, + DepositAcxPriceConsumer, + SuggestedFeesConsumer, + TrackFillEventConsumer, + ]; + + return { + module: ScraperModule, + providers, + imports: [ + Web3Module, + AppConfigModule, + TypeOrmModule.forFeature([ProcessedBlock, MerkleDistributorProcessedBlock, Claim, Deposit]), + MarketPriceModule.forRoot(moduleOptions), + HttpModule, + DepositModule.forRoot(moduleOptions), + AirdropModule.forRoot(moduleOptions), + ReferralModule.forRoot(moduleOptions), + BullModule.registerQueue({ + name: ScraperQueue.BlockNumber, + }), + BullModule.registerQueue({ + name: ScraperQueue.BlocksEvents, + }), + BullModule.registerQueue({ + name: ScraperQueue.MerkleDistributorBlocksEvents, + }), + BullModule.registerQueue({ + name: ScraperQueue.TokenDetails, + }), + BullModule.registerQueue({ + name: ScraperQueue.TokenPrice, + }), + BullModule.registerQueue({ + name: ScraperQueue.DepositReferral, + }), + BullModule.registerQueue({ + name: ScraperQueue.DepositAcxPrice, + }), + BullModule.registerQueue({ + name: ScraperQueue.FillEvents, + defaultJobOptions: { + backoff: 120 * 1000, + attempts: Number.MAX_SAFE_INTEGER, + removeOnComplete: true, + removeOnFail: true, + }, + }), + BullModule.registerQueue({ + name: ScraperQueue.SpeedUpEvents, + defaultJobOptions: { + backoff: 120 * 1000, + attempts: Number.MAX_SAFE_INTEGER, + removeOnComplete: true, + removeOnFail: true, + }, + }), + BullModule.registerQueue({ + name: ScraperQueue.DepositFilledDate, + }), + BullModule.registerQueue({ + name: ScraperQueue.SuggestedFees, + }), + BullModule.registerQueue({ + name: ScraperQueue.TrackFillEvent, + }), + ], + exports: [ScraperQueuesService], + controllers: [ScraperController], + }; + } +} diff --git a/src/modules/scraper/service/ScraperQueuesService.ts b/src/modules/scraper/service/ScraperQueuesService.ts index 8ec9c95c..33841135 100644 --- a/src/modules/scraper/service/ScraperQueuesService.ts +++ b/src/modules/scraper/service/ScraperQueuesService.ts @@ -59,7 +59,10 @@ export class ScraperQueuesService { setInterval(() => { for (const queueName of Object.keys(this.queuesMap)) { const queue = this.queuesMap[queueName] as Queue; - queue.getJobCounts().then((data) => this.logger.log(`${queueName} ${JSON.stringify(data)}`)); + queue + .getJobCounts() + .then((data) => this.logger.log(`${queueName} ${JSON.stringify(data)}`)) + .catch((data) => this.logger.error(`${queueName} ${JSON.stringify(data)}`)); } }, 1000 * 60); } diff --git a/src/modules/user/module.ts b/src/modules/user/module.ts index 40e6384b..95731769 100644 --- a/src/modules/user/module.ts +++ b/src/modules/user/module.ts @@ -1,4 +1,4 @@ -import { Module } from "@nestjs/common"; +import { Module, Provider, DynamicModule } from "@nestjs/common"; import { TypeOrmModule } from "@nestjs/typeorm"; import { UserController } from "./entry-points/http/user.controller"; import { User } from "./model/user.entity"; @@ -7,11 +7,30 @@ import { UserService } from "./services/user.service"; import { UserWalletService } from "./services/user-wallet.service"; import { UserFixture } from "./adapter/db/user-fixture"; import { UserWalletFixture } from "./adapter/db/user-wallet-fixture"; +import { ModuleOptions, RunMode } from "../../dynamic-module"; -@Module({ - providers: [UserService, UserWalletService, UserFixture, UserWalletFixture], - controllers: [UserController], - imports: [TypeOrmModule.forFeature([User, UserWallet])], - exports: [UserService, UserWalletService], -}) -export class UserModule {} +@Module({}) +export class UserModule { + static forRoot(moduleOptions: ModuleOptions): DynamicModule { + let module: DynamicModule = { module: UserModule, providers: [], controllers: [], imports: [], exports: [] }; + + if (moduleOptions.runModes.includes(RunMode.Normal) || moduleOptions.runModes.includes(RunMode.Test)) { + module = { + ...module, + controllers: [...module.controllers, UserController], + providers: [...module.providers, UserService, UserWalletService], + imports: [...module.imports, TypeOrmModule.forFeature([User, UserWallet])], + exports: [UserService, UserWalletService], + }; + } + + if (moduleOptions.runModes.includes(RunMode.Test)) { + module = { + ...module, + providers: [...module.providers, UserFixture, UserWalletFixture], + }; + } + + return module; + } +} diff --git a/test/airdrop.e2e-spec.ts b/test/airdrop.e2e-spec.ts index 6c299874..2ce939d2 100644 --- a/test/airdrop.e2e-spec.ts +++ b/test/airdrop.e2e-spec.ts @@ -6,7 +6,7 @@ import { JwtService } from "@nestjs/jwt"; import { ValidationPipe } from "../src/validation.pipe"; import { AppModule } from "../src/app.module"; import { GetAirdropRewardsResponse } from "../src/modules/airdrop/entry-points/http/dto"; -import { DepositFixture, mockDepositEntity } from "../src/modules/scraper/adapter/db/deposit-fixture"; +import { DepositFixture, mockDepositEntity } from "../src/modules/deposit/adapter/db/deposit-fixture"; import { Role } from "../src/modules/auth/entry-points/http/roles"; import { WalletRewardsFixture } from "../src/modules/airdrop/adapter/db/wallet-rewards-fixture"; import { CommunityRewardsFixture } from "../src/modules/airdrop/adapter/db/community-rewards-fixture"; @@ -15,12 +15,13 @@ import { UserWalletFixture } from "../src/modules/user/adapter/db/user-wallet-fi import { configValues } from "../src/modules/configuration"; import { TokenFixture } from "../src/modules/web3/adapters/db/token-fixture"; import { BigNumber } from "ethers"; +import { RunMode } from "../src/dynamic-module"; let app: INestApplication; beforeAll(async () => { const moduleFixture = await Test.createTestingModule({ - imports: [AppModule], + imports: [AppModule.forRoot({ runModes: [RunMode.Normal, RunMode.Test, RunMode.Scraper] })], }).compile(); app = moduleFixture.createNestApplication(); @@ -199,7 +200,7 @@ describe("POST /airdrop/upload/rewards", () => { }); it("should not be authorized if not admin", async () => { - const userJwt = app.get(JwtService).sign({ roles: [Role.User] }, { secret: configValues().auth.jwtSecret }); + const userJwt = app.get(JwtService).sign({ roles: [Role.User] }); const response = await request(app.getHttpServer()) .post("/airdrop/upload/rewards") .set({ Authorization: `Bearer ${userJwt}` }); diff --git a/test/airdrop/merkle-distributor.e2e-spec.ts b/test/airdrop/merkle-distributor.e2e-spec.ts index 1ae00f55..9575741f 100644 --- a/test/airdrop/merkle-distributor.e2e-spec.ts +++ b/test/airdrop/merkle-distributor.e2e-spec.ts @@ -10,6 +10,7 @@ import { MerkleDistributorWindowFixture } from "../../src/modules/airdrop/adapte import { MerkleDistributorRecipientFixture } from "../../src/modules/airdrop/adapter/db/merkle-distributor-recipient"; import { UserFixture } from "../../src/modules/user/adapter/db/user-fixture"; import { UserWalletFixture } from "../../src/modules/user/adapter/db/user-wallet-fixture"; +import { RunMode } from "../../src/dynamic-module"; let app: INestApplication; let merkleDistributorWindowFixture: MerkleDistributorWindowFixture; @@ -19,7 +20,7 @@ let userWalletFixture: UserWalletFixture; beforeAll(async () => { const moduleFixture = await Test.createTestingModule({ - imports: [AppModule], + imports: [AppModule.forRoot({ runModes: [RunMode.Normal, RunMode.Test, RunMode.Scraper] })], }).compile(); app = moduleFixture.createNestApplication(); diff --git a/test/deposit.e2e-spec.ts b/test/deposit.e2e-spec.ts index 05d7b825..8fd293a7 100644 --- a/test/deposit.e2e-spec.ts +++ b/test/deposit.e2e-spec.ts @@ -3,15 +3,16 @@ import { INestApplication } from "@nestjs/common"; import { Test } from "@nestjs/testing"; import { constants } from "ethers"; -import { DepositFixture, mockManyDepositEntities } from "../src/modules/scraper/adapter/db/deposit-fixture"; +import { DepositFixture, mockManyDepositEntities } from "../src/modules/deposit/adapter/db/deposit-fixture"; import { ValidationPipe } from "../src/validation.pipe"; import { AppModule } from "../src/app.module"; +import { RunMode } from "../src/dynamic-module"; let app: INestApplication; beforeAll(async () => { const moduleFixture = await Test.createTestingModule({ - imports: [AppModule], + imports: [AppModule.forRoot({ runModes: [RunMode.Normal, RunMode.Test, RunMode.Scraper] })], }).compile(); app = moduleFixture.createNestApplication(); diff --git a/test/referrals.e2e-spec.ts b/test/referrals.e2e-spec.ts index 7db843e7..e7ef3844 100644 --- a/test/referrals.e2e-spec.ts +++ b/test/referrals.e2e-spec.ts @@ -4,8 +4,8 @@ import { JwtService } from "@nestjs/jwt"; import { Test } from "@nestjs/testing"; import { ethers, utils } from "ethers"; -import { DepositFixture, mockDepositEntity } from "../src/modules/scraper/adapter/db/deposit-fixture"; -import { ClaimFixture } from "../src/modules/scraper/adapter/db/claim-fixture"; +import { DepositFixture, mockDepositEntity } from "../src/modules/deposit/adapter/db/deposit-fixture"; +import { ClaimFixture } from "../src/modules/airdrop/adapter/db/claim-fixture"; import { TokenFixture } from "../src/modules/web3/adapters/db/token-fixture"; import { HistoricMarketPriceFixture } from "../src/modules/market-price/adapters/hmp-fixture"; import { AppModule } from "../src/app.module"; @@ -14,6 +14,7 @@ import { Token } from "../src/modules/web3/model/token.entity"; import { HistoricMarketPrice } from "../src/modules/market-price/model/historic-market-price.entity"; import { Role } from "../src/modules/auth/entry-points/http/roles"; import { configValues } from "../src/modules/configuration"; +import { RunMode } from "../src/dynamic-module"; const referrer = "0x9A8f92a830A5cB89a3816e3D267CB7791c16b04D"; const depositor = "0xdf120Bf3AEE9892f213B1Ba95035a60682D637c3"; @@ -39,7 +40,7 @@ let adminJwt: string; beforeAll(async () => { const moduleFixture = await Test.createTestingModule({ - imports: [AppModule], + imports: [AppModule.forRoot({ runModes: [RunMode.Normal, RunMode.Test, RunMode.Scraper] })], }).compile(); app = moduleFixture.createNestApplication(); diff --git a/test/run-modes.e2e-spec.ts b/test/run-modes.e2e-spec.ts new file mode 100644 index 00000000..48452132 --- /dev/null +++ b/test/run-modes.e2e-spec.ts @@ -0,0 +1,30 @@ +import { Test } from "@nestjs/testing"; +import request from "supertest"; +import { AppModule } from "../src/app.module"; +import { RunMode } from "../src/dynamic-module"; + +describe("Run modes", () => { + it("should start app in normal mode", async () => { + const moduleFixture = await Test.createTestingModule({ + imports: [AppModule.forRoot({ runModes: [RunMode.Normal] })], + }).compile(); + + const app = moduleFixture.createNestApplication(); + await app.init(); + const response = await request(app.getHttpServer()).get("/health"); + expect(response.statusCode).toEqual(200); + await app.close(); + }); + + it("should start app in scraper mode", async () => { + const moduleFixture = await Test.createTestingModule({ + imports: [AppModule.forRoot({ runModes: [RunMode.Scraper] })], + }).compile(); + + const app = moduleFixture.createNestApplication(); + await app.init(); + const response = await request(app.getHttpServer()).get("/health"); + expect(response.statusCode).toEqual(200); + await app.close(); + }); +}); diff --git a/test/scraper.e2e-spec.ts b/test/scraper.e2e-spec.ts index 0717f37e..a55c7d83 100644 --- a/test/scraper.e2e-spec.ts +++ b/test/scraper.e2e-spec.ts @@ -1,17 +1,18 @@ import { INestApplication } from "@nestjs/common"; import { Test } from "@nestjs/testing"; -import { DepositFixture } from "../src/modules/scraper/adapter/db/deposit-fixture"; +import { DepositFixture } from "../src/modules/deposit/adapter/db/deposit-fixture"; import { AppModule } from "../src/app.module"; import { QueryFailedError } from "typeorm"; import { FillEventsConsumer } from "../src/modules/scraper/adapter/messaging/FillEventsConsumer"; import { FillEventsQueueMessage } from "../src/modules/scraper/adapter/messaging"; +import { RunMode } from "../src/dynamic-module"; describe("Scraper module", () => { let app: INestApplication; beforeAll(async () => { const moduleFixture = await Test.createTestingModule({ - imports: [AppModule], + imports: [AppModule.forRoot({ runModes: [RunMode.Normal, RunMode.Test, RunMode.Scraper] })], }).compile(); app = moduleFixture.createNestApplication(); diff --git a/test/user.e2e-spec.ts b/test/user.e2e-spec.ts index 5032c249..24c4b499 100644 --- a/test/user.e2e-spec.ts +++ b/test/user.e2e-spec.ts @@ -1,16 +1,17 @@ import request from "supertest"; import { INestApplication } from "@nestjs/common"; import { Test } from "@nestjs/testing"; +import { JwtService } from "@nestjs/jwt"; import { Wallet, constants } from "ethers"; import { ValidationPipe } from "../src/validation.pipe"; import { AppModule } from "../src/app.module"; -import { generateJwtForUser } from "./utils"; import { UserService } from "../src/modules/user/services/user.service"; import { User } from "../src/modules/user/model/user.entity"; import { UserFixture } from "../src/modules/user/adapter/db/user-fixture"; import { getRandomInt } from "../src/utils"; import { UserWalletFixture } from "../src/modules/user/adapter/db/user-wallet-fixture"; +import { RunMode } from "../src/dynamic-module"; const signer = Wallet.createRandom(); const nonExistingUser = { @@ -34,7 +35,7 @@ let userWalletFixture: UserWalletFixture; beforeAll(async () => { const testingModule = await Test.createTestingModule({ - imports: [AppModule], + imports: [AppModule.forRoot({ runModes: [RunMode.Normal, RunMode.Test, RunMode.Scraper] })], }).compile(); app = testingModule.createNestApplication(); @@ -56,9 +57,9 @@ beforeAll(async () => { discordName: "name1", }), ]); - validJwtForExistingUser = generateJwtForUser(existingUser); - validJwtForExistingUser2 = generateJwtForUser(existingUser2); - validJwtForNonExistingUser = generateJwtForUser(nonExistingUser); + validJwtForExistingUser = app.get(JwtService).sign({ id: existingUser.id }); + validJwtForExistingUser2 = app.get(JwtService).sign({ id: existingUser2.id }); + validJwtForNonExistingUser = app.get(JwtService).sign({ id: nonExistingUser.id }); [validSignatureForExistingUser, validSignatureForNonExistingUser] = await Promise.all([ signer.signMessage(existingUser.discordId), signer.signMessage(nonExistingUser.discordId), diff --git a/test/utils.ts b/test/utils.ts deleted file mode 100644 index f48197f4..00000000 --- a/test/utils.ts +++ /dev/null @@ -1,7 +0,0 @@ -import { sign, SignOptions } from "jsonwebtoken"; -import { configValues } from "../src/modules/configuration/index"; - -export function generateJwtForUser(user: { id: number; shortId: string; uuid: string }, jwtOptions?: SignOptions) { - const payload = { id: user.id, uuid: user.uuid, shortId: user.shortId }; - return sign(payload, configValues().auth.jwtSecret, jwtOptions); -}