Merge pull request #261 from swisstopo/feature/assets-247-asset-viewer

Automatically sync search index

apps/client-asset-sg/src/app/components/menu-bar/menu-bar.component.html

@@ -69,7 +69,7 @@
     <svg-icon key="profile" />
     <span translate>menuBar.profile</span>
   </a>
-  <a routerLink="/help" disabled asset-sg-reset class="menu-bar-item">
+  <a aria-disabled="true" [disabled]="true" asset-sg-reset class="menu-bar-item">
     <svg-icon key="help" />
     <span translate>menuBar.help</span>
   </a>

apps/server-asset-sg/src/app.module.ts

@@ -18,6 +18,7 @@ import { AssetsController } from '@/features/assets/assets.controller';
 import { AssetSearchController } from '@/features/assets/search/asset-search.controller';
 import { AssetSearchService } from '@/features/assets/search/asset-search.service';
 import { AssetSyncController } from '@/features/assets/sync/asset-sync.controller';
+import { AssetSyncService } from '@/features/assets/sync/asset-sync.service';
 import { ContactRepo } from '@/features/contacts/contact.repo';
 import { ContactsController } from '@/features/contacts/contacts.controller';
 import { FavoriteRepo } from '@/features/favorites/favorite.repo';
@@ -54,6 +55,7 @@ import { WorkgroupsController } from '@/features/workgroups/workgroups.controlle
     AssetInfoRepo,
     AssetRepo,
     AssetSearchService,
+    AssetSyncService,
     ContactRepo,
     FavoriteRepo,
     PrismaService,

apps/server-asset-sg/src/features/assets/asset.repo.ts

@@ -14,6 +14,10 @@ import { handlePrismaMutationError } from '@/utils/prisma';
 export class AssetRepo implements FindRepo<Asset, AssetId>, MutateRepo<Asset, AssetId, FullAssetData> {
   constructor(private readonly prisma: PrismaService) {}
 
+  async count() {
+    return await this.prisma.asset.count();
+  }
+
   async find(id: AssetId): Promise<Asset | null> {
     const entry = await this.prisma.asset.findFirst({
       where: { assetId: id },

apps/server-asset-sg/src/features/assets/search/asset-search.service.ts

@@ -64,6 +64,10 @@ export class AssetSearchService {
     return this.registerWithOptions(oneOrMore, { index: INDEX, shouldRefresh: true });
   }
 
+  async count(): Promise<number> {
+    return (await this.elastic.count({ index: INDEX })).count;
+  }
+
   async syncWithDatabase(onProgress?: (percentage: number) => void | Promise<void>): Promise<void> {
     // Write all Prisma assets into the sync index.
     const total = await this.prisma.asset.count();

apps/server-asset-sg/src/features/assets/sync/asset-sync.controller.ts

@@ -1,64 +1,32 @@
-import fs from 'fs/promises';
-
-import { Controller, Get, HttpException, OnApplicationBootstrap, Post, Res } from '@nestjs/common';
+import { Controller, Get, HttpException, Post, Res } from '@nestjs/common';
 import { Response } from 'express';
 import { Authorize } from '@/core/decorators/authorize.decorator';
-import { AssetSearchService } from '@/features/assets/search/asset-search.service';
+import { AssetSyncService } from '@/features/assets/sync/asset-sync.service';
 
 @Controller('/assets/sync')
-export class AssetSyncController implements OnApplicationBootstrap {
-  constructor(private readonly assetSearchService: AssetSearchService) {}
-
-  async onApplicationBootstrap() {
-    const syncFileExists = await fs
-      .access(assetSyncFile)
-      .then(() => true)
-      .catch(() => false);
-    if (syncFileExists) {
-      void fs.rm(assetSyncFile);
-    }
-  }
+export class AssetSyncController {
+  constructor(private readonly assetSyncService: AssetSyncService) {}
 
   @Get('/')
   @Authorize.Admin()
   async show(@Res() res: Response): Promise<{ progress: number } | void> {
     try {
-      const data = await fs.readFile(assetSyncFile, { encoding: 'utf-8' });
-      const state: AssetSyncState = JSON.parse(data);
-      res.status(200).json({ progress: state.progress }).end();
-    } catch (e) {
-      if ((e as { code?: string }).code === 'ENOENT') {
+      const state = await this.assetSyncService.show();
+      if (state === null) {
         res.status(204).end();
         return;
       }
+      res.status(200).json({ progress: state.progress }).end();
+    } catch (e) {
       throw new HttpException(`${e}`, 500);
     }
   }
 
   @Post('/')
   @Authorize.Admin()
   async start(@Res() res: Response): Promise<void> {
-    const isSyncRunning = await fs
-      .access(assetSyncFile)
-      .then(() => true)
-      .catch(() => false);
-    if (isSyncRunning) {
-      res.status(204).end();
-      return;
-    }
-
-    const writeProgress = (progress: number): Promise<void> => {
-      const state: AssetSyncState = { progress: parseFloat(progress.toFixed(3)) };
-      const data = JSON.stringify(state);
-      return fs.writeFile(assetSyncFile, data, { encoding: 'utf-8' });
-    };
-
-    await writeProgress(0);
-    setTimeout(async () => {
-      await this.assetSearchService.syncWithDatabase(writeProgress);
-      await fs.rm(assetSyncFile);
-    });
-    res.status(201).end();
+    await this.assetSyncService.start();
+    res.status(200).end();
   }
 }
 

apps/server-asset-sg/src/features/assets/sync/asset-sync.service.ts

@@ -0,0 +1,90 @@
+import fs from 'fs/promises';
+import { Injectable, OnApplicationBootstrap } from '@nestjs/common';
+import { SchedulerRegistry } from '@nestjs/schedule';
+import { CronJob } from 'cron';
+import { AssetRepo } from '@/features/assets/asset.repo';
+import { AssetSearchService } from '@/features/assets/search/asset-search.service';
+
+@Injectable()
+export class AssetSyncService implements OnApplicationBootstrap {
+  constructor(
+    private readonly assetSearchService: AssetSearchService,
+    private readonly schedulerRegistry: SchedulerRegistry,
+    private readonly assetRepo: AssetRepo
+  ) {}
+  async onApplicationBootstrap() {
+    const syncFileExists = await this.isSyncRunning();
+    if (syncFileExists) {
+      void fs.rm(assetSyncFile);
+    }
+
+    if (process.env.ANONYMOUS_MODE === 'true') {
+      console.log('Anonymous Mode is activated. Search Index will be automatically synced.');
+      await this.startSyncIfIndexOutOfSync();
+
+      const every20Minutes = '*/20 * * * *';
+      const job = new CronJob(every20Minutes, () => this.startSyncIfIndexOutOfSync());
+      this.schedulerRegistry.addCronJob('elasticIndexSync', job);
+      job.start();
+    }
+  }
+
+  async show(): Promise<AssetSyncState | null> {
+    try {
+      const data = await fs.readFile(assetSyncFile, { encoding: 'utf-8' });
+      return JSON.parse(data);
+    } catch (e) {
+      if ((e as { code?: string }).code === 'ENOENT') {
+        return null;
+      }
+      throw e;
+    }
+  }
+
+  async isSyncRunning() {
+    return await fs
+      .access(assetSyncFile)
+      .then(() => true)
+      .catch(() => false);
+  }
+
+  async start(): Promise<void> {
+    if (await this.isSyncRunning()) {
+      console.debug('AssetSyncService.start: Sync already running.');
+      return;
+    }
+
+    const writeProgress = (progress: number): Promise<void> => {
+      const state: AssetSyncState = { progress: parseFloat(progress.toFixed(3)) };
+      const data = JSON.stringify(state);
+      return fs.writeFile(assetSyncFile, data, { encoding: 'utf-8' });
+    };
+
+    await writeProgress(0);
+    setTimeout(async () => {
+      await this.assetSearchService.syncWithDatabase(writeProgress);
+      await fs.rm(assetSyncFile);
+    });
+  }
+
+  private async startSyncIfIndexOutOfSync() {
+    console.debug(`startSyncIfIndexOutOfSync.`);
+    const numberOfAssets = await this.assetRepo.count();
+    const numberOfIndexedAssets = await this.assetSearchService.count();
+    console.debug(`Found ${numberOfAssets} Assets and ${numberOfIndexedAssets} Indexed documents.`);
+    if (numberOfAssets !== numberOfIndexedAssets) {
+      await this.start();
+    }
+  }
+}
+
+/**
+ * The file into which the progress of the current asset sync is written.
+ * This allows the sync to be shared across all users and requests without requiring a database entry.
+ * Note that the file path is relative to the project root, _not_ to this file.
+ */
+const assetSyncFile = './asset-sync-progress.tmp.json';
+
+interface AssetSyncState {
+  progress: number;
+}

package.json

@@ -61,6 +61,7 @@
     "cache-manager": "^5.4.0",
     "class-transformer": "^0.5.1",
     "class-validator": "^0.14.1",
+    "cron": "^3.1.7",
     "csv-parse": "^5.3.3",
     "fp-ts": "^2.13.1",
     "io-ts": "^2.2.20",