From 9884800b0613706ec3d7bd2925463e0f6c57b892 Mon Sep 17 00:00:00 2001 From: Jannic Veith Date: Thu, 29 Aug 2024 16:27:57 +0200 Subject: [PATCH 1/4] Disable help icon --- .../src/app/components/menu-bar/menu-bar.component.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/client-asset-sg/src/app/components/menu-bar/menu-bar.component.html b/apps/client-asset-sg/src/app/components/menu-bar/menu-bar.component.html index a0eed6b5..6bc281ab 100644 --- a/apps/client-asset-sg/src/app/components/menu-bar/menu-bar.component.html +++ b/apps/client-asset-sg/src/app/components/menu-bar/menu-bar.component.html @@ -69,7 +69,7 @@ menuBar.profile - + menuBar.help From 13334b22639fe5e66864d6c7f8a9cccf3e54d9ea Mon Sep 17 00:00:00 2001 From: Jannic Veith Date: Fri, 30 Aug 2024 09:03:43 +0200 Subject: [PATCH 2/4] Add automatic Sync Service for View --- apps/server-asset-sg/.env | 2 +- apps/server-asset-sg/src/app.module.ts | 2 + .../src/features/assets/asset.repo.ts | 4 + .../assets/search/asset-search.service.ts | 4 + .../assets/sync/asset-sync.controller.ts | 52 +++-------- .../assets/sync/asset-sync.service.ts | 90 +++++++++++++++++++ package-lock.json | 1 + package.json | 1 + 8 files changed, 113 insertions(+), 43 deletions(-) create mode 100644 apps/server-asset-sg/src/features/assets/sync/asset-sync.service.ts diff --git a/apps/server-asset-sg/.env b/apps/server-asset-sg/.env index b4ffa40b..2579410c 100644 --- a/apps/server-asset-sg/.env +++ b/apps/server-asset-sg/.env @@ -12,7 +12,7 @@ OAUTH_SCOPE=openid profile email cognito OAUTH_SHOW_DEBUG_INFO=true OAUTH_TOKEN_ENDPOINT=http://localhost:4011/connect/token OAUTH_AUTHORIZED_GROUPS=assets.swissgeol -ANONYMOUS_MODE=false +ANONYMOUS_MODE=true OCR_URL= OCR_CALLBACK_URL= diff --git a/apps/server-asset-sg/src/app.module.ts b/apps/server-asset-sg/src/app.module.ts index a52e1fa3..abd27cfe 100644 --- a/apps/server-asset-sg/src/app.module.ts +++ b/apps/server-asset-sg/src/app.module.ts @@ -18,6 +18,7 @@ import { AssetsController } from '@/features/assets/assets.controller'; import { AssetSearchController } from '@/features/assets/search/asset-search.controller'; import { AssetSearchService } from '@/features/assets/search/asset-search.service'; import { AssetSyncController } from '@/features/assets/sync/asset-sync.controller'; +import { AssetSyncService } from '@/features/assets/sync/asset-sync.service'; import { ContactRepo } from '@/features/contacts/contact.repo'; import { ContactsController } from '@/features/contacts/contacts.controller'; import { FavoriteRepo } from '@/features/favorites/favorite.repo'; @@ -54,6 +55,7 @@ import { WorkgroupsController } from '@/features/workgroups/workgroups.controlle AssetInfoRepo, AssetRepo, AssetSearchService, + AssetSyncService, ContactRepo, FavoriteRepo, PrismaService, diff --git a/apps/server-asset-sg/src/features/assets/asset.repo.ts b/apps/server-asset-sg/src/features/assets/asset.repo.ts index c616982d..4ad86e6e 100644 --- a/apps/server-asset-sg/src/features/assets/asset.repo.ts +++ b/apps/server-asset-sg/src/features/assets/asset.repo.ts @@ -14,6 +14,10 @@ import { handlePrismaMutationError } from '@/utils/prisma'; export class AssetRepo implements FindRepo, MutateRepo { constructor(private readonly prisma: PrismaService) {} + async count() { + return await this.prisma.asset.count(); + } + async find(id: AssetId): Promise { const entry = await this.prisma.asset.findFirst({ where: { assetId: id }, diff --git a/apps/server-asset-sg/src/features/assets/search/asset-search.service.ts b/apps/server-asset-sg/src/features/assets/search/asset-search.service.ts index bcce17a8..9f7afd8f 100644 --- a/apps/server-asset-sg/src/features/assets/search/asset-search.service.ts +++ b/apps/server-asset-sg/src/features/assets/search/asset-search.service.ts @@ -64,6 +64,10 @@ export class AssetSearchService { return this.registerWithOptions(oneOrMore, { index: INDEX, shouldRefresh: true }); } + async count(): Promise { + return (await this.elastic.count({ index: INDEX })).count; + } + async syncWithDatabase(onProgress?: (percentage: number) => void | Promise): Promise { // Write all Prisma assets into the sync index. const total = await this.prisma.asset.count(); diff --git a/apps/server-asset-sg/src/features/assets/sync/asset-sync.controller.ts b/apps/server-asset-sg/src/features/assets/sync/asset-sync.controller.ts index f4efc305..fbd90206 100644 --- a/apps/server-asset-sg/src/features/assets/sync/asset-sync.controller.ts +++ b/apps/server-asset-sg/src/features/assets/sync/asset-sync.controller.ts @@ -1,36 +1,23 @@ -import fs from 'fs/promises'; - -import { Controller, Get, HttpException, OnApplicationBootstrap, Post, Res } from '@nestjs/common'; +import { Controller, Get, HttpException, Post, Res } from '@nestjs/common'; import { Response } from 'express'; import { Authorize } from '@/core/decorators/authorize.decorator'; -import { AssetSearchService } from '@/features/assets/search/asset-search.service'; +import { AssetSyncService } from '@/features/assets/sync/asset-sync.service'; @Controller('/assets/sync') -export class AssetSyncController implements OnApplicationBootstrap { - constructor(private readonly assetSearchService: AssetSearchService) {} - - async onApplicationBootstrap() { - const syncFileExists = await fs - .access(assetSyncFile) - .then(() => true) - .catch(() => false); - if (syncFileExists) { - void fs.rm(assetSyncFile); - } - } +export class AssetSyncController { + constructor(private readonly assetSyncService: AssetSyncService) {} @Get('/') @Authorize.Admin() async show(@Res() res: Response): Promise<{ progress: number } | void> { try { - const data = await fs.readFile(assetSyncFile, { encoding: 'utf-8' }); - const state: AssetSyncState = JSON.parse(data); - res.status(200).json({ progress: state.progress }).end(); - } catch (e) { - if ((e as { code?: string }).code === 'ENOENT') { + const state = await this.assetSyncService.show(); + if (state === null) { res.status(204).end(); return; } + res.status(200).json({ progress: state.progress }).end(); + } catch (e) { throw new HttpException(`${e}`, 500); } } @@ -38,27 +25,8 @@ export class AssetSyncController implements OnApplicationBootstrap { @Post('/') @Authorize.Admin() async start(@Res() res: Response): Promise { - const isSyncRunning = await fs - .access(assetSyncFile) - .then(() => true) - .catch(() => false); - if (isSyncRunning) { - res.status(204).end(); - return; - } - - const writeProgress = (progress: number): Promise => { - const state: AssetSyncState = { progress: parseFloat(progress.toFixed(3)) }; - const data = JSON.stringify(state); - return fs.writeFile(assetSyncFile, data, { encoding: 'utf-8' }); - }; - - await writeProgress(0); - setTimeout(async () => { - await this.assetSearchService.syncWithDatabase(writeProgress); - await fs.rm(assetSyncFile); - }); - res.status(201).end(); + await this.assetSyncService.start(); + res.status(200).end(); } } diff --git a/apps/server-asset-sg/src/features/assets/sync/asset-sync.service.ts b/apps/server-asset-sg/src/features/assets/sync/asset-sync.service.ts new file mode 100644 index 00000000..9469f808 --- /dev/null +++ b/apps/server-asset-sg/src/features/assets/sync/asset-sync.service.ts @@ -0,0 +1,90 @@ +import fs from 'fs/promises'; +import { Injectable, OnApplicationBootstrap } from '@nestjs/common'; +import { SchedulerRegistry } from '@nestjs/schedule'; +import { CronJob } from 'cron'; +import { AssetRepo } from '@/features/assets/asset.repo'; +import { AssetSearchService } from '@/features/assets/search/asset-search.service'; + +@Injectable() +export class AssetSyncService implements OnApplicationBootstrap { + constructor( + private readonly assetSearchService: AssetSearchService, + private readonly schedulerRegistry: SchedulerRegistry, + private readonly assetRepo: AssetRepo + ) {} + async onApplicationBootstrap() { + const syncFileExists = await this.isSyncRunning(); + if (syncFileExists) { + void fs.rm(assetSyncFile); + } + + if (process.env.ANONYMOUS_MODE === 'true') { + console.log('Anonymous Mode is activated. Search Index will be automatically synced.'); + await this.startSyncIfIndexOutOfSync(); + + const every20Minutes = '*/20 * * * *'; + const job = new CronJob(every20Minutes, () => this.startSyncIfIndexOutOfSync()); + this.schedulerRegistry.addCronJob('elasticIndexSync', job); + job.start(); + } + } + + async show(): Promise { + try { + const data = await fs.readFile(assetSyncFile, { encoding: 'utf-8' }); + return JSON.parse(data); + } catch (e) { + if ((e as { code?: string }).code === 'ENOENT') { + return null; + } + throw e; + } + } + + async isSyncRunning() { + return await fs + .access(assetSyncFile) + .then(() => true) + .catch(() => false); + } + + async start(): Promise { + if (await this.isSyncRunning()) { + console.debug('AssetSyncService.start: Sync already running.'); + return; + } + + const writeProgress = (progress: number): Promise => { + const state: AssetSyncState = { progress: parseFloat(progress.toFixed(3)) }; + const data = JSON.stringify(state); + return fs.writeFile(assetSyncFile, data, { encoding: 'utf-8' }); + }; + + await writeProgress(0); + setTimeout(async () => { + await this.assetSearchService.syncWithDatabase(writeProgress); + await fs.rm(assetSyncFile); + }); + } + + private async startSyncIfIndexOutOfSync() { + console.debug(`startSyncIfIndexOutOfSync.`); + const numberOfAssets = await this.assetRepo.count(); + const numberOfIndixedAssets = await this.assetSearchService.count(); + console.debug(`Found ${numberOfAssets} Assets and ${numberOfIndixedAssets} Indexed documents.`); + if (numberOfAssets !== numberOfIndixedAssets) { + await this.start(); + } + } +} + +/** + * The file into which the progress of the current asset sync is written. + * This allows the sync to be shared across all users and requests without requiring a database entry. + * Note that the file path is relative to the project root, _not_ to this file. + */ +const assetSyncFile = './asset-sync-progress.tmp.json'; + +interface AssetSyncState { + progress: number; +} diff --git a/package-lock.json b/package-lock.json index 1470d055..633ee7cf 100644 --- a/package-lock.json +++ b/package-lock.json @@ -50,6 +50,7 @@ "cache-manager": "^5.4.0", "class-transformer": "^0.5.1", "class-validator": "^0.14.1", + "cron": "^3.1.7", "csv-parse": "^5.3.3", "fp-ts": "^2.13.1", "io-ts": "^2.2.20", diff --git a/package.json b/package.json index 10b30679..02f1cf82 100644 --- a/package.json +++ b/package.json @@ -61,6 +61,7 @@ "cache-manager": "^5.4.0", "class-transformer": "^0.5.1", "class-validator": "^0.14.1", + "cron": "^3.1.7", "csv-parse": "^5.3.3", "fp-ts": "^2.13.1", "io-ts": "^2.2.20", From 14421c5c36d18dad7e0ec8678be407bd31a27b7c Mon Sep 17 00:00:00 2001 From: Jannic Veith Date: Mon, 2 Sep 2024 11:55:10 +0200 Subject: [PATCH 3/4] Revert unwanted env change --- apps/server-asset-sg/.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/server-asset-sg/.env b/apps/server-asset-sg/.env index 2579410c..b4ffa40b 100644 --- a/apps/server-asset-sg/.env +++ b/apps/server-asset-sg/.env @@ -12,7 +12,7 @@ OAUTH_SCOPE=openid profile email cognito OAUTH_SHOW_DEBUG_INFO=true OAUTH_TOKEN_ENDPOINT=http://localhost:4011/connect/token OAUTH_AUTHORIZED_GROUPS=assets.swissgeol -ANONYMOUS_MODE=true +ANONYMOUS_MODE=false OCR_URL= OCR_CALLBACK_URL= From 242bbd02c245ead3f3ab74918158e50087763270 Mon Sep 17 00:00:00 2001 From: Jannic Veith Date: Tue, 3 Sep 2024 09:24:27 +0200 Subject: [PATCH 4/4] Fix typo --- .../src/features/assets/sync/asset-sync.service.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/server-asset-sg/src/features/assets/sync/asset-sync.service.ts b/apps/server-asset-sg/src/features/assets/sync/asset-sync.service.ts index 9469f808..fc70db3c 100644 --- a/apps/server-asset-sg/src/features/assets/sync/asset-sync.service.ts +++ b/apps/server-asset-sg/src/features/assets/sync/asset-sync.service.ts @@ -70,9 +70,9 @@ export class AssetSyncService implements OnApplicationBootstrap { private async startSyncIfIndexOutOfSync() { console.debug(`startSyncIfIndexOutOfSync.`); const numberOfAssets = await this.assetRepo.count(); - const numberOfIndixedAssets = await this.assetSearchService.count(); - console.debug(`Found ${numberOfAssets} Assets and ${numberOfIndixedAssets} Indexed documents.`); - if (numberOfAssets !== numberOfIndixedAssets) { + const numberOfIndexedAssets = await this.assetSearchService.count(); + console.debug(`Found ${numberOfAssets} Assets and ${numberOfIndexedAssets} Indexed documents.`); + if (numberOfAssets !== numberOfIndexedAssets) { await this.start(); } }