From e3d18f7fe32500c3b3316d96febd2f4dd224dd19 Mon Sep 17 00:00:00 2001 From: Daniel von Atzigen Date: Fri, 5 Jul 2024 07:25:02 +0200 Subject: [PATCH 1/2] Replace `GET /api/all-study` with more perfomant `GET /api/studies` --- apps/server-asset-sg/src/app.module.ts | 4 ++ .../features/asset-old/asset.controller.ts | 5 -- .../src/features/asset-old/asset.service.ts | 35 +-------- .../src/features/assets/asset.model.ts | 17 ++--- .../src/features/assets/asset.repo.ts | 16 ++--- .../src/features/assets/assets.http | 4 ++ .../src/features/assets/prisma-asset.ts | 10 +-- .../features/studies/studies.controller.ts | 72 +++++++++++++++++++ .../src/features/studies/studies.http | 4 ++ .../src/features/studies/study.model.ts | 16 +++++ .../src/features/studies/study.repo.ts | 64 +++++++++++++++++ .../src/lib/services/all-study.service.ts | 57 +++++++++++---- libs/shared/src/lib/models/all-study.ts | 8 ++- 13 files changed, 235 insertions(+), 77 deletions(-) create mode 100644 apps/server-asset-sg/src/features/studies/studies.controller.ts create mode 100644 apps/server-asset-sg/src/features/studies/studies.http create mode 100644 apps/server-asset-sg/src/features/studies/study.model.ts create mode 100644 apps/server-asset-sg/src/features/studies/study.repo.ts diff --git a/apps/server-asset-sg/src/app.module.ts b/apps/server-asset-sg/src/app.module.ts index f0523f8b..28307e9f 100644 --- a/apps/server-asset-sg/src/app.module.ts +++ b/apps/server-asset-sg/src/app.module.ts @@ -24,6 +24,8 @@ import { ContactsController } from '@/features/contacts/contacts.controller'; import { FavoriteRepo } from '@/features/favorites/favorite.repo'; import { FavoritesController } from '@/features/favorites/favorites.controller'; import { OcrController } from '@/features/ocr/ocr.controller'; +import { StudiesController } from '@/features/studies/studies.controller'; +import { StudyRepo } from '@/features/studies/study.repo'; import { UserRepo } from '@/features/users/user.repo'; import { UsersController } from '@/features/users/users.controller'; @@ -36,6 +38,7 @@ import { UsersController } from '@/features/users/users.controller'; AssetSearchController, AssetsController, AssetController, + StudiesController, ContactsController, OcrController, ], @@ -50,6 +53,7 @@ import { UsersController } from '@/features/users/users.controller'; ContactRepo, FavoriteRepo, UserRepo, + StudyRepo, AssetEditService, AssetSearchService, { diff --git a/apps/server-asset-sg/src/features/asset-old/asset.controller.ts b/apps/server-asset-sg/src/features/asset-old/asset.controller.ts index 6fcbf0dd..78097c65 100644 --- a/apps/server-asset-sg/src/features/asset-old/asset.controller.ts +++ b/apps/server-asset-sg/src/features/asset-old/asset.controller.ts @@ -67,11 +67,6 @@ export class AssetController { return e.right; } - @Get('/all-study') - getAllStudies() { - return this.assetService.getAllStudies(); - } - @Get('/reference-data') async getReferenceData() { const e = await this.assetService.getReferenceData()(); diff --git a/apps/server-asset-sg/src/features/asset-old/asset.service.ts b/apps/server-asset-sg/src/features/asset-old/asset.service.ts index 8d2b7d53..2adc4d41 100644 --- a/apps/server-asset-sg/src/features/asset-old/asset.service.ts +++ b/apps/server-asset-sg/src/features/asset-old/asset.service.ts @@ -7,6 +7,8 @@ import { SearchAssetResult, UsageCode, makeUsageCode, + AllStudyDTOFromAPI, + AllStudyRaw, } from '@asset-sg/shared'; import { Injectable } from '@nestjs/common'; import { sequenceS } from 'fp-ts/Apply'; @@ -34,39 +36,6 @@ import { postgresStudiesByAssetId } from '@/utils/postgres-studies/postgres-stud export class AssetService { constructor(private readonly prismaService: PrismaService, private readonly assetSearchService: AssetSearchService) {} - async getData(polygon: [number, number][]) { - const polygonParam = polygon.map((point) => `${point[0]} ${point[1]}`).join(','); - - const result = await this.prismaService.$queryRawUnsafe( - `select id, accident_uid, year, month, canton, ST_AsText(geom) as geom from bicycle_accidents where ST_INTERSECTS(geom, ST_GeomFromText('POLYGON((${polygonParam}))', 2056))` - ); - return { result }; - } - - async getAllStudies() { - interface RawStudy { - studyId: string; - assetId: number; - isPoint: boolean; - centroidGeomText: string; - } - const rawData: RawStudy[] = await this.prismaService.$queryRawUnsafe(` - SELECT - study_id AS "studyId", - asset_id AS "assetId", - is_point AS "isPoint", - centroid_geom_text AS "centroidGeomText" - FROM public.all_study - `); - - return rawData.map((study) => ({ - studyId: study.studyId, - assetId: study.assetId, - isPoint: study.isPoint, - centroid: study.centroidGeomText.replace('POINT(', '').replace(')', ''), - })); - } - getFile(fileId: number) { return getFile(this.prismaService, fileId); } diff --git a/apps/server-asset-sg/src/features/assets/asset.model.ts b/apps/server-asset-sg/src/features/assets/asset.model.ts index 9e6c8a02..cf4baf13 100644 --- a/apps/server-asset-sg/src/features/assets/asset.model.ts +++ b/apps/server-asset-sg/src/features/assets/asset.model.ts @@ -2,6 +2,7 @@ import { Transform, Type } from 'class-transformer'; import { IsArray, IsBoolean, IsDate, IsEnum, IsInt, IsObject, IsString, ValidateNested } from 'class-validator'; import { IsNullable, messageNullableInt, messageNullableString } from '@/core/decorators/is-nullable.decorator'; +import { StudyType } from '@/features/studies/study.model'; import { LocalDate } from '@/utils/data/local-date'; import { Data, Model } from '@/utils/data/model'; @@ -54,7 +55,7 @@ export interface AssetDetails { infoGeol: InfoGeol; usage: AssetUsages; statuses: WorkStatus[]; - studies: Study[]; + studies: AssetStudy[]; } export interface AssetUsages { @@ -70,7 +71,7 @@ export interface AssetData extends Omit, NonDataKeys> { links: AssetLinksData; identifiers: (AssetIdentifier | AssetIdentifierData)[]; statuses: (WorkStatus | WorkStatusData)[]; - studies: (Study | StudyData)[]; + studies: (AssetStudy | StudyData)[]; } interface InfoGeol { @@ -137,20 +138,14 @@ export enum UsageCode { UseOnRequest = 'useOnRequest', } -export interface Study extends Model { +export interface AssetStudy extends Model { geom: string; type: StudyType; } -export type StudyData = Data; +export type StudyData = Data; -export type StudyId = number; - -export enum StudyType { - Area = 'area', - Location = 'location', - Trace = 'trace', -} +export type AssetStudyId = number; export class AssetUsageBoundary implements AssetUsage { @IsBoolean() diff --git a/apps/server-asset-sg/src/features/assets/asset.repo.ts b/apps/server-asset-sg/src/features/assets/asset.repo.ts index ace93487..c331094e 100644 --- a/apps/server-asset-sg/src/features/assets/asset.repo.ts +++ b/apps/server-asset-sg/src/features/assets/asset.repo.ts @@ -8,12 +8,12 @@ import { AssetData, AssetId, AssetUsage, - Study, + AssetStudy, StudyData, - StudyId, - StudyType, + AssetStudyId, } from '@/features/assets/asset.model'; import { assetSelection, parseAssetFromPrisma } from '@/features/assets/prisma-asset'; +import { StudyType } from '@/features/studies/study.model'; import { User } from '@/features/users/user.model'; import { isNotPersisted, isPersisted } from '@/utils/data/model'; import { satisfy } from '@/utils/define'; @@ -130,9 +130,9 @@ export class AssetRepo implements Repo { } } - private async manageStudies(assetId: AssetId, data: (Study | StudyData)[]): Promise { + private async manageStudies(assetId: AssetId, data: (AssetStudy | StudyData)[]): Promise { const studiesToCreate: Partial> = {}; - const studiesToUpdate: Partial> = {}; + const studiesToUpdate: Partial> = {}; for (const study of data) { if (isPersisted(study)) { (studiesToUpdate[study.type] ??= []).push(study); @@ -140,7 +140,7 @@ export class AssetRepo implements Repo { (studiesToCreate[study.type] ??= []).push(study); } } - for (const [type, studies] of Object.entries(studiesToUpdate) as Array<[StudyType, Study[]]>) { + for (const [type, studies] of Object.entries(studiesToUpdate) as Array<[StudyType, AssetStudy[]]>) { await this.deleteStudies( assetId, type, @@ -153,7 +153,7 @@ export class AssetRepo implements Repo { } } - private async deleteStudies(assetId: AssetId, type: StudyType, knownIds: StudyId[]): Promise { + private async deleteStudies(assetId: AssetId, type: StudyType, knownIds: AssetStudyId[]): Promise { const condition = knownIds.length === 0 ? '' : Prisma.sql`AND study_${Prisma.raw(type)}_id NOT IN (${Prisma.join(knownIds, ',')})`; await this.prisma.$queryRaw` @@ -183,7 +183,7 @@ export class AssetRepo implements Repo { `; } - private async updateStudies(assetId: AssetId, type: StudyType, data: Study[]): Promise { + private async updateStudies(assetId: AssetId, type: StudyType, data: AssetStudy[]): Promise { if (data.length === 0) { return; } diff --git a/apps/server-asset-sg/src/features/assets/assets.http b/apps/server-asset-sg/src/features/assets/assets.http index e04c5563..96b462de 100644 --- a/apps/server-asset-sg/src/features/assets/assets.http +++ b/apps/server-asset-sg/src/features/assets/assets.http @@ -87,3 +87,7 @@ Content-Type: application/json } } } + +### List studies +GET {{host}}/api/all-study-short +Authorization: Impersonate {{user}} diff --git a/apps/server-asset-sg/src/features/assets/prisma-asset.ts b/apps/server-asset-sg/src/features/assets/prisma-asset.ts index b2d84d2d..f09428be 100644 --- a/apps/server-asset-sg/src/features/assets/prisma-asset.ts +++ b/apps/server-asset-sg/src/features/assets/prisma-asset.ts @@ -6,11 +6,11 @@ import { AssetUsage, ContactAssignmentRole, LinkedAsset, - Study, - StudyId, - StudyType, + AssetStudy, + AssetStudyId, UsageStatusCode, } from '@/features/assets/asset.model'; +import { StudyType } from '@/features/studies/study.model'; import { LocalDate } from '@/utils/data/local-date'; import { satisfy } from '@/utils/define'; @@ -192,7 +192,7 @@ export const parseAssetFromPrisma = (data: SelectedAsset): Asset => ({ id, type, geom: it.geomText, - } as Study; + } as AssetStudy; }), }); @@ -207,7 +207,7 @@ const parseUsage = (data: SelectedUsage): AssetUsage => ({ availableAt: data.startAvailabilityDate == null ? null : LocalDate.fromDate(data.startAvailabilityDate), }); -const parseStudyId = (studyId: string): { type: StudyType; id: StudyId } => { +const parseStudyId = (studyId: string): { type: StudyType; id: AssetStudyId } => { if (!studyId.startsWith('study_')) { throw new Error('expected studyId to start with `study_`'); } diff --git a/apps/server-asset-sg/src/features/studies/studies.controller.ts b/apps/server-asset-sg/src/features/studies/studies.controller.ts new file mode 100644 index 00000000..851d12de --- /dev/null +++ b/apps/server-asset-sg/src/features/studies/studies.controller.ts @@ -0,0 +1,72 @@ +import { Readable } from 'stream'; +import { Controller, Get, Res } from '@nestjs/common'; +import { Response } from 'express'; +import { RequireRole } from '@/core/decorators/require-role.decorator'; +import { Study } from '@/features/studies/study.model'; +import { StudyRepo } from '@/features/studies/study.repo'; +import { Role } from '@/features/users/user.model'; + +@Controller('/studies') +export class StudiesController { + constructor(private readonly studyRepo: StudyRepo) {} + + @Get('/') + @RequireRole(Role.Viewer) + async list(@Res() res: Response): Promise { + // This route loads all studies and encodes them as CSV. + // CSV has been chosen as we have a large amount of studies (13'000+) + // and need a concise format that can be processed in batches (which, for example, JSON can't). + + res.setHeader('Content-Type', 'text/csv'); + res.setHeader('Transfer-Encoding', 'chunked'); + + // The size of the first load. + // This should be a relatively small number, as unlike all subsequent batches, + // the first batch can't be run in parallel with a response write. + const INITIAL_BATCH_SIZE = 500; + + // The size of the second and all subsequent batches. + // This value should be chosen so that it approximately evens out the duration + // of a database read and a response write of a batch, as they are run in parallel. + const BATCH_SIZE = 2_500; + + const { studyRepo } = this; + async function* load() { + // The amount of studies that have been read up to now. + let count = 0; + + // The promise that is loading the next batch. + // Note that this is running in parallel to the response writer. + let next: Promise | null = studyRepo.list({ limit: INITIAL_BATCH_SIZE, offset: 0 }); + + // The maximal size of the next batch. + let nextLimit = INITIAL_BATCH_SIZE; + + // Load batches until we don't load a new one. + while (next != null) { + // Wait for the database read to complete. + const studies: Study[] = await next; + + // Add the amount of studies to the total counter. + count += studies.length; + + // We only start a new database read if we haven't loaded all existing studies yet. + next = + studies.length === 0 || studies.length < nextLimit + ? null + : studyRepo.list({ limit: BATCH_SIZE, offset: count }); + + // Update the `nextLimit`, as in the first iteration, it's set to INITIAL_BATCH_SIZE. + nextLimit = BATCH_SIZE; + + // Write the current batch to the response. + for (const study of studies) { + yield `${study.id.substring(6)};${study.assetId};${+study.isPoint};${study.center.x};${study.center.y}\n`; + } + } + } + + const stream = Readable.from(load()); + stream.pipe(res); + } +} diff --git a/apps/server-asset-sg/src/features/studies/studies.http b/apps/server-asset-sg/src/features/studies/studies.http new file mode 100644 index 00000000..f60d3fc5 --- /dev/null +++ b/apps/server-asset-sg/src/features/studies/studies.http @@ -0,0 +1,4 @@ +### List studies +GET {{host}}/api/studies +Authorization: Impersonate {{user}} +Accept: text/csv diff --git a/apps/server-asset-sg/src/features/studies/study.model.ts b/apps/server-asset-sg/src/features/studies/study.model.ts new file mode 100644 index 00000000..1b636695 --- /dev/null +++ b/apps/server-asset-sg/src/features/studies/study.model.ts @@ -0,0 +1,16 @@ +import { LV95 } from '@asset-sg/shared'; +import { AssetId } from '@/features/assets/asset.model'; + +export interface Study { + id: StudyId; + center: LV95; + isPoint: boolean; + assetId: AssetId; +} + +export type StudyId = `study_${StudyType}_${number}`; +export enum StudyType { + Area = 'area', + Location = 'location', + Trace = 'trace', +} diff --git a/apps/server-asset-sg/src/features/studies/study.repo.ts b/apps/server-asset-sg/src/features/studies/study.repo.ts new file mode 100644 index 00000000..26899860 --- /dev/null +++ b/apps/server-asset-sg/src/features/studies/study.repo.ts @@ -0,0 +1,64 @@ +import { LV95, LV95FromSpaceSeparatedString } from '@asset-sg/shared'; +import { Injectable } from '@nestjs/common'; +import { Prisma } from '@prisma/client'; +import * as E from 'fp-ts/Either'; +import { PrismaService } from '@/core/prisma.service'; +import { ReadRepo, RepoListOptions } from '@/core/repo'; +import { Study, StudyId } from '@/features/studies/study.model'; + +@Injectable() +export class StudyRepo implements ReadRepo { + constructor(private readonly prisma: PrismaService) {} + + async find(id: StudyId): Promise { + const result = await this.query(Prisma.sql` + WHERE + study_id = ${id} + LIMIT 1 + `); + return result.length === 1 ? result[0] : null; + } + + list({ limit, offset, ids }: RepoListOptions = {}): Promise { + const conditions: Prisma.Sql[] = []; + if (ids != null && ids.length > 0) { + conditions.push(Prisma.sql` + WHERE study_id IN (${Prisma.join(ids, ',')}) + `); + } + conditions.push(Prisma.sql` + ORDER BY asset_id + `); + if (limit != null) { + conditions.push(Prisma.sql` + LIMIT ${limit} + `); + } + if (offset != null && offset !== 0) { + conditions.push(Prisma.sql` + OFFSET ${offset} + `); + } + return this.query(Prisma.join(conditions, ' ')); + } + + private async query(condition: Prisma.Sql): Promise { + type RawStudy = Omit & { center: string }; + const studies: RawStudy[] = await this.prisma.$queryRaw` + SELECT + study_id as "id", + asset_id AS "assetId", + is_point AS "isPoint", + SUBSTRING(centroid_geom_text FROM 7 FOR length(centroid_geom_text) -7) AS "center" + FROM public.all_study + ${condition} + `; + return studies.map((study) => { + const center = LV95FromSpaceSeparatedString.decode(study.center); + return { + ...study, + center: (center as E.Right).right, + }; + }); + } +} diff --git a/libs/asset-viewer/src/lib/services/all-study.service.ts b/libs/asset-viewer/src/lib/services/all-study.service.ts index 04e730fd..fbe997b0 100644 --- a/libs/asset-viewer/src/lib/services/all-study.service.ts +++ b/libs/asset-viewer/src/lib/services/all-study.service.ts @@ -1,14 +1,13 @@ -import { HttpClient, HttpErrorResponse } from '@angular/common/http'; +import { HttpClient, HttpDownloadProgressEvent, HttpEvent, HttpEventType } from '@angular/common/http'; import { Injectable } from '@angular/core'; -import { ApiError, httpErrorResponseError } from '@asset-sg/client-shared'; -import { OE, ORD, decodeError, unknownToUnknownError } from '@asset-sg/core'; -import { AllStudyDTOsFromAPI } from '@asset-sg/shared'; +import { ApiError } from '@asset-sg/client-shared'; +import { ORD } from '@asset-sg/core'; +import { LV95 } from '@asset-sg/shared'; import * as RD from '@devexperts/remote-data-ts'; import * as E from 'fp-ts/Either'; -import { flow } from 'fp-ts/function'; -import { map, startWith } from 'rxjs'; +import { concatMap, filter, from, map, Observable, scan, share, toArray } from 'rxjs'; -import { AllStudyDTOs } from '../models'; +import { AllStudyDTO, AllStudyDTOs } from '../models'; @Injectable({ providedIn: 'root' }) export class AllStudyService { @@ -18,14 +17,44 @@ export class AllStudyService { return this.getAllStudiesFromApi(); } - private getAllStudiesFromApi(): ORD.ObservableRemoteData { - return this._httpClient.get('/api/all-study').pipe( - map(flow(AllStudyDTOsFromAPI.decode, E.mapLeft(decodeError))), - OE.catchErrorW((err: HttpErrorResponse | unknown) => - err instanceof HttpErrorResponse ? httpErrorResponseError(err) : unknownToUnknownError(err) - ), + private getAllStudiesFromApi(): ORD.ObservableRemoteData { + return this._httpClient.get('/api/studies', { observe: 'events', responseType: 'text', reportProgress: true }).pipe( + filter((event) => event.type === HttpEventType.DownloadProgress || event.type === HttpEventType.Response), + map((event: HttpEvent) => (event as HttpDownloadProgressEvent).partialText ?? ''), + bufferUntilLineEnd(), + filter((line) => line.length !== 0), + map((line) => { + const [id, assetId, isPoint, x, y] = line.split(';'); + return { + studyId: `study_${id}`, + assetId: parseInt(assetId), + isPoint: Boolean(isPoint), + centroid: { x: parseInt(x), y: parseInt(y) } as LV95, + } as AllStudyDTO; + }), + toArray(), + map((it) => E.right(it)), map(RD.fromEither), - startWith(RD.pending) + share() ); } } + +function bufferUntilLineEnd() { + return (source: Observable) => + source.pipe( + scan( + (acc, chunk) => { + acc.buffer += chunk.substring(acc.previous.length); + const lines = acc.buffer.split('\n'); + acc.buffer = lines.pop() ?? ''; // Keep the last partial line in the buffer + acc.lines = lines; + acc.previous = chunk; + return acc; + }, + { buffer: '', previous: '', lines: [] as string[] } + ), + filter((acc) => acc.lines.length > 0), + concatMap((acc) => from(acc.lines)) + ); +} diff --git a/libs/shared/src/lib/models/all-study.ts b/libs/shared/src/lib/models/all-study.ts index d185fe43..60377213 100644 --- a/libs/shared/src/lib/models/all-study.ts +++ b/libs/shared/src/lib/models/all-study.ts @@ -10,4 +10,10 @@ export const AllStudyDTOFromAPI = D.struct({ }); export type AllStudyDTOFromAPI = D.TypeOf; export const AllStudyDTOsFromAPI = D.array(AllStudyDTOFromAPI); -export type AllStudyDTOsFromAPI = D.TypeOf; + +export interface AllStudyRaw { + studyId: string; + assetId: number; + isPoint: boolean; + centroid: string; +} From 5bfbc69acae6bb20fa54d723cf4b2fe9607056d9 Mon Sep 17 00:00:00 2001 From: Daniel von Atzigen Date: Fri, 5 Jul 2024 13:19:51 +0200 Subject: [PATCH 2/2] Move polygon search to Elasticsearch Remove unused code Replace fp-ts backed `LV95FromSpaceSeparatedString` with `parseLV95` Extract and test `serializeStudyAsCsv` --- .prettierignore | 1 + .../src/features/asset-old/asset.service.ts | 208 +----------------- .../assets/search/asset-search.controller.ts | 13 +- .../features/assets/search/asset-search.http | 12 +- .../search/asset-search.service.spec.ts | 4 +- .../assets/search/asset-search.service.ts | 206 ++++++++++------- .../features/studies/studies.controller.ts | 5 +- .../src/features/studies/study.model.spec.ts | 36 +++ .../src/features/studies/study.model.ts | 4 + .../src/features/studies/study.repo.ts | 5 +- .../mappings/swissgeol_asset_asset.json | 3 + .../src/lib/services/asset-search.service.ts | 2 +- libs/shared/src/lib/models/all-study.ts | 8 - .../src/lib/models/elastic-search-asset.ts | 9 + libs/shared/src/lib/models/lv95.ts | 8 + package-lock.json | 31 +++ package.json | 22 +- 17 files changed, 255 insertions(+), 322 deletions(-) create mode 100644 apps/server-asset-sg/src/features/studies/study.model.spec.ts diff --git a/.prettierignore b/.prettierignore index 24c4e11b..8f8e33c6 100644 --- a/.prettierignore +++ b/.prettierignore @@ -7,3 +7,4 @@ /dist /node_modules /tmp +/.idea diff --git a/apps/server-asset-sg/src/features/asset-old/asset.service.ts b/apps/server-asset-sg/src/features/asset-old/asset.service.ts index 2adc4d41..ffbb93b4 100644 --- a/apps/server-asset-sg/src/features/asset-old/asset.service.ts +++ b/apps/server-asset-sg/src/features/asset-old/asset.service.ts @@ -1,25 +1,11 @@ -import { DT, decodeError, isNotNil, unknownToError, unknownToUnknownError } from '@asset-sg/core'; -import { - AssetSearchParams, - BaseAssetDetail, - DateId, - DateIdFromDate, - SearchAssetResult, - UsageCode, - makeUsageCode, - AllStudyDTOFromAPI, - AllStudyRaw, -} from '@asset-sg/shared'; +import { decodeError, isNotNil, unknownToError, unknownToUnknownError } from '@asset-sg/core'; +import { AssetSearchParams, BaseAssetDetail, SearchAssetResult } from '@asset-sg/shared'; import { Injectable } from '@nestjs/common'; import { sequenceS } from 'fp-ts/Apply'; import * as A from 'fp-ts/Array'; -import { contramap } from 'fp-ts/Eq'; -import { Lazy, flow, pipe } from 'fp-ts/function'; -import * as NEA from 'fp-ts/NonEmptyArray'; -import * as N from 'fp-ts/number'; +import { flow, Lazy, pipe } from 'fp-ts/function'; import * as O from 'fp-ts/Option'; import * as RR from 'fp-ts/ReadonlyRecord'; -import * as S from 'fp-ts/string'; import * as TE from 'fp-ts/TaskEither'; import * as C from 'io-ts/Codec'; import * as D from 'io-ts/Decoder'; @@ -79,114 +65,6 @@ export class AssetService { ); } - // findAssetsByPolygon(polygon: [number, number][]) { - // return findAssetsByPolygon(this.prismaService, polygon); - // } - - // findAssetsByPolygon(polygon: [number, number][]) { - // const polygonParam = polygon.map(point => `${point[0]} ${point[1]}`).join(','); - - // return pipe( - // TE.tryCatch( - // () => - // this.prismaService.$queryRawUnsafe(` - // select - // a.asset_id as "assetId", - // a.title_public as "titlePublic", - // a.create_date as "createDateId", - // a.asset_kind_item_code as "assetKindItemCode", - // a.asset_format_item_code as "assetFormatItemCode", - // mclr.man_cat_label_item_code as "manCatLabelItemCode", - // ac.contact_id as "contactId", - // ac.role as "contactRole", - // s.study_id as "studyId", - // s.geom_text as "studyGeomText" - // from all_study s - // inner join asset a - // on s.asset_id = a.asset_id - // left join - // asset_contact ac - // on ac.asset_id = a.asset_id - // left join - // man_cat_label_ref mclr - // on ac.asset_id = mclr.asset_id - // where - // st_intersects(geom, st_geomfromtext('polygon((${polygonParam}))', 2056)) - // order by - // a.asset_id - // `), - // unknownToError, - // ), - // TE.chainW( - // flow( - // DBResultList.decode, - // E.mapLeft(e => new Error(D.draw(e))), - // TE.fromEither, - // ), - // ), - // TE.map( - // flow( - // NEA.fromArray, - // O.map(assets => { - // const orderedDates: NEA.NonEmptyArray = pipe( - // assets, - // NEA.map(a => a.createDate), - // NEA.uniq(DateIdOrd), - // NEA.sort(DateIdOrd), - // ); - // return SearchAssetResult.encode({ - // _tag: 'SearchAssetResultNonEmpty', - // aggregations: { - // ranges: { createDate: { min: NEA.head(orderedDates), max: NEA.last(orderedDates) } }, - // buckets: { - // authorIds: pipe( - // assets, - // A.map(a => a.contacts.filter(c => c.role === 'author').map(c => c.id)), - // A.flatten, - // NEA.fromArray, - // O.map( - // flow( - // NEA.groupBy(a => String(a)), - // R.map(g => ({ key: NEA.head(g), count: g.length })), - // R.toArray, - // A.map(([, value]) => value), - // ), - // ), - // O.getOrElseW(() => []), - // ), - // }, - // }, - // assets: assets.map(asset => ({ ...asset, score: 1 })), - // }); - // }), - // O.getOrElse(() => SearchAssetResult.encode({ _tag: 'SearchAssetResultEmpty' })), - // ), - // ), - // ); - // } - - // async findStudiesByPolygon(polygon: [number, number][]) { - // const polygonParam = polygon.map(point => `${point[0]} ${point[1]}`).join(','); - - // console.log( - // `select study_id as "studyId", geom_text as "geomText", asset_id as "assetId" from public.all_study where st_intersects(geom, st_geomfromtext('polygon((${polygonParam}))', 2056))`, - // ); - - // return await this.prismaService.$queryRawUnsafe( - // `select study_id as "studyId", geom_text as "geomText", asset_id as "assetId" - // from public.all_study - // where st_intersects(geom, st_geomfromtext('polygon((${polygonParam}))', 2056))`, - // ); - // } - - // async findStudiesByAssetId(assetId: number) { - // return await this.prismaService.$queryRawUnsafe( - // `select study_id as "studyId", geom_text as "geomText", asset_id as "assetId" - // from public.all_study - // where asset_id=${assetId}}`, - // ); - // } - getReferenceData() { const qt = (f: Lazy>, key: K, newKey: string) => pipe( @@ -276,83 +154,3 @@ export class AssetService { ); } } - -const DBResultRaw = D.struct({ - assetId: D.number, - titlePublic: D.string, - contactId: DT.optionFromNullable(D.number), - contactRole: DT.optionFromNullable(D.string), - createDate: DateIdFromDate, - assetKindItemCode: D.string, - assetFormatItemCode: D.string, - languageItemCode: D.string, - manCatLabelItemCode: DT.optionFromNullable(D.string), - internalUse: D.boolean, - publicUse: D.boolean, - studyId: DT.optionFromNullable(D.string), - studyGeomText: DT.optionFromNullable(D.string), -}); -type DBResultRaw = D.TypeOf; - -// eslint-disable-next-line @typescript-eslint/no-unused-vars -const DBResultRawList = D.array(DBResultRaw); - -type DBResultRawList = D.TypeOf; - -type DBResult = { - assetId: number; - titlePublic: string; - createDate: DateId; - assetKindItemCode: string; - assetFormatItemCode: string; - languageItemCode: string; - manCatLabelItemCodes: Array; - usageCode: UsageCode; - contacts: Array<{ role: string; id: number }>; - studies: Array<{ studyId: string; geomText: string }>; -}; - -const eqStudyByStudyId = contramap((x: { studyId: string; geomText: string }) => x.studyId)(S.Eq); -const eqContactByContactId = contramap((x: { contactId: number; contactRole: string }) => x.contactId)(N.Eq); - -// eslint-disable-next-line @typescript-eslint/no-unused-vars -const dbResultRawToDBResult = (dbResultRawList: NEA.NonEmptyArray): DBResult => { - const { - assetId, - titlePublic, - createDate, - assetFormatItemCode, - assetKindItemCode, - languageItemCode, - internalUse, - publicUse, - } = NEA.head(dbResultRawList); - return { - assetId, - titlePublic, - createDate, - assetFormatItemCode, - assetKindItemCode, - languageItemCode, - contacts: pipe( - dbResultRawList, - NEA.map((x) => sequenceS(O.Apply)({ contactId: x.contactId, contactRole: x.contactRole })), - A.compact, - A.uniq(eqContactByContactId), - A.map((a) => ({ id: a.contactId, role: a.contactRole })) - ), - studies: pipe( - dbResultRawList, - NEA.map((x) => sequenceS(O.Apply)({ studyId: x.studyId, geomText: x.studyGeomText })), - A.compact, - A.uniq(eqStudyByStudyId) - ), - manCatLabelItemCodes: pipe( - dbResultRawList, - NEA.map((x) => x.manCatLabelItemCode), - A.compact, - A.uniq(S.Eq) - ), - usageCode: makeUsageCode(publicUse, internalUse), - }; -}; diff --git a/apps/server-asset-sg/src/features/assets/search/asset-search.controller.ts b/apps/server-asset-sg/src/features/assets/search/asset-search.controller.ts index efc10a13..6d21584e 100644 --- a/apps/server-asset-sg/src/features/assets/search/asset-search.controller.ts +++ b/apps/server-asset-sg/src/features/assets/search/asset-search.controller.ts @@ -1,14 +1,12 @@ import { - AssetEditDetail, AssetSearchQueryDTO, AssetSearchResult, AssetSearchResultDTO, AssetSearchStats, AssetSearchStatsDTO, } from '@asset-sg/shared'; -import { Body, Controller, Post, Query, ValidationPipe } from '@nestjs/common'; +import { Body, Controller, HttpCode, HttpStatus, Post, Query, ValidationPipe } from '@nestjs/common'; import { plainToInstance } from 'class-transformer'; - import { AssetSearchService } from '@/features/assets/search/asset-search.service'; @Controller('/assets/search') @@ -16,6 +14,7 @@ export class AssetSearchController { constructor(private readonly assetSearchService: AssetSearchService) {} @Post('/') + @HttpCode(HttpStatus.OK) async search( @Body(new ValidationPipe({ transform: true, whitelist: true, forbidNonWhitelisted: true })) query: AssetSearchQueryDTO, @@ -28,14 +27,12 @@ export class AssetSearchController { ): Promise { limit = limit == null ? limit : Number(limit); offset = offset == null ? offset : Number(offset); - const result = await this.assetSearchService.search(query, { limit, offset }); - return plainToInstance(AssetSearchResultDTO, { - ...result, - data: result.data.map(AssetEditDetail.encode) as unknown as AssetEditDetail[], - }); + const result = await this.assetSearchService.search(query, { limit, offset, decode: false }); + return plainToInstance(AssetSearchResultDTO, result); } @Post('/stats') + @HttpCode(HttpStatus.OK) async showStats( @Body(new ValidationPipe({ transform: true, whitelist: true, forbidNonWhitelisted: true })) query: AssetSearchQueryDTO diff --git a/apps/server-asset-sg/src/features/assets/search/asset-search.http b/apps/server-asset-sg/src/features/assets/search/asset-search.http index 8f0daa16..37df4b73 100644 --- a/apps/server-asset-sg/src/features/assets/search/asset-search.http +++ b/apps/server-asset-sg/src/features/assets/search/asset-search.http @@ -1,18 +1,10 @@ -### Start asset sync -POST {{host}}/api/assets/sync -Authorization: Impersonate {{user}} - -### Show asset sync progress -GET {{host}}/api/assets/sync -Authorization: Impersonate {{user}} - ### Search assets -POST {{host}}/api/assets/search?limit=1 +POST {{host}}/api/assets/search?limit=10000 Content-Type: application/json Authorization: Impersonate {{user}} { - "languageItemCodes": ["DE"] + "text": "n1" } ### Show assets search stats diff --git a/apps/server-asset-sg/src/features/assets/search/asset-search.service.spec.ts b/apps/server-asset-sg/src/features/assets/search/asset-search.service.spec.ts index ce5a99dc..d5738e2f 100644 --- a/apps/server-asset-sg/src/features/assets/search/asset-search.service.spec.ts +++ b/apps/server-asset-sg/src/features/assets/search/asset-search.service.spec.ts @@ -32,12 +32,14 @@ import { PrismaService } from '@/core/prisma.service'; import { fakeAssetPatch, fakeAssetUsage, fakeContact, fakeUser } from '@/features/asset-old/asset-edit.fake'; import { AssetData, AssetEditRepo } from '@/features/asset-old/asset-edit.repo'; import { AssetEditDetail } from '@/features/asset-old/asset-edit.service'; +import { StudyRepo } from '@/features/studies/study.repo'; describe(AssetSearchService, () => { const elastic = openElasticsearchClient(); const prisma = new PrismaService(); const assetRepo = new AssetEditRepo(prisma); - const service = new AssetSearchService(elastic, prisma, assetRepo); + const studyRepo = new StudyRepo(prisma); + const service = new AssetSearchService(elastic, prisma, assetRepo, studyRepo); beforeAll(async () => { const existsIndex = await elastic.indices.exists({ index: ASSET_ELASTIC_INDEX }); diff --git a/apps/server-asset-sg/src/features/assets/search/asset-search.service.ts b/apps/server-asset-sg/src/features/assets/search/asset-search.service.ts index 694cb19b..35fc35b8 100644 --- a/apps/server-asset-sg/src/features/assets/search/asset-search.service.ts +++ b/apps/server-asset-sg/src/features/assets/search/asset-search.service.ts @@ -1,20 +1,23 @@ /* eslint-disable @typescript-eslint/no-non-null-assertion */ import { AssetByTitle, + AssetEditDetail, AssetSearchQuery, AssetSearchResult, AssetSearchStats, + dateFromDateId, DateId, + dateIdFromDate, + ElasticPoint, ElasticSearchAsset, GeometryCode, - Polygon, + LV95, + makeUsageCode, SearchAssetAggregations, SearchAssetResult, + SerializedAssetEditDetail, UsageCode, ValueCount, - dateFromDateId, - dateIdFromDate, - makeUsageCode, } from '@asset-sg/shared'; import { Client as ElasticsearchClient } from '@elastic/elasticsearch'; import { @@ -24,26 +27,28 @@ import { SearchTotalHits, } from '@elastic/elasticsearch/lib/api/types'; import { Injectable } from '@nestjs/common'; -import { Prisma } from '@prisma/client'; +import * as E from 'fp-ts/Either'; +import proj4 from 'proj4'; // eslint-disable-next-line @nx/enforce-module-boundaries import indexMapping from '../../../../../../development/init/elasticsearch/mappings/swissgeol_asset_asset.json'; +import { AssetId } from '../asset.model'; import { PrismaService } from '@/core/prisma.service'; import { AssetEditRepo } from '@/features/asset-old/asset-edit.repo'; -import { AssetEditDetail } from '@/features/asset-old/asset-edit.service'; -import { AssetEditDetailFromPostgres } from '@/models/asset-edit-detail'; +import { StudyId } from '@/features/studies/study.model'; +import { StudyRepo } from '@/features/studies/study.repo'; const INDEX = 'swissgeol_asset_asset'; export { INDEX as ASSET_ELASTIC_INDEX }; interface SearchOptions { scope: Array; - assetIds?: number[]; + assetIds?: AssetId[]; } interface ElasticSearchResult { - scoresByAssetId: Map; + scoresByAssetId: Map; aggs: SearchAssetAggregations; } @@ -52,7 +57,8 @@ export class AssetSearchService { constructor( private readonly elastic: ElasticsearchClient, private readonly prisma: PrismaService, - private readonly assetRepo: AssetEditRepo + private readonly assetRepo: AssetEditRepo, + private readonly studyRepo: StudyRepo ) {} register(oneOrMore: AssetEditDetail | AssetEditDetail[]): Promise { @@ -105,7 +111,7 @@ export class AssetSearchService { ...indexMapping, }); - // Refresh the sync index so we can reindex its contents. + // Refresh the sync index, so we can reindex its contents. await this.elastic.indices.refresh({ index: SYNC_INDEX }); // Copy the sync index's contents into the empty asset index. @@ -126,11 +132,16 @@ export class AssetSearchService { { index, shouldRefresh = false }: { index: string; shouldRefresh?: boolean } ): Promise { const assets = Array.isArray(oneOrMore) ? oneOrMore : [oneOrMore]; - const elasticAssets = await Promise.all(assets.map((asset) => this.mapAssetToElastic(asset))); - const operations = elasticAssets.reduce((ops, elasticAsset) => { - ops.push({ index: { _index: index, _id: `${elasticAsset.assetId}` } }, elasticAsset); - return ops; - }, [] as Array); + const operations: Array = Array(assets.length * 2); + const mappings: Array> = Array(assets.length); + for (let i = 0; i < assets.length; i++) { + const asset = assets[i]; + mappings[i] = this.mapAssetToElastic(asset).then((elasticAsset) => { + operations[i * 2] = { index: { _index: index, _id: `${elasticAsset.assetId}` } }; + operations[i * 2 + 1] = elasticAsset; + }); + } + await Promise.all(mappings); await this.elastic.bulk({ index, refresh: shouldRefresh, @@ -152,20 +163,29 @@ export class AssetSearchService { * @param query The query to match with. * @param limit The maximum amount of assets to load. Defaults to `100`. * @param offset The amount of assets being skipped before loading the assets. + * @param decode Whether to decode the assets. If this is set to `false`, the assets should be decoded via `AssetEditDetail` before accessing them. + * This option primarily exists so that the assets are not decoded and directly re-encoded when returning them via API. */ - async search(query: AssetSearchQuery, { limit = 100, offset = 0 }: PageOptions = {}): Promise { + async search( + query: AssetSearchQuery, + { limit = 100, offset = 0, decode = true }: PageOptions & { decode?: boolean } = {} + ): Promise { // Apply the query to find all matching ids. - const ids = await this.searchIds(query); + const [serializedAssets, total] = await this.searchAssetsByQuery(query, { limit, offset }); // Load the matched assets from the database. - const data = await this.assetRepo.list({ ids, limit, offset }); + const data: AssetEditDetail[] = []; + for (const serializedAsset of serializedAssets.values()) { + const asset = JSON.parse(serializedAsset); + data.push(decode ? (AssetEditDetail.decode(asset) as E.Right).right : asset); + } // Return the matched data in a paginated format. return { page: { offset, size: data.length, - total: ids.length, + total, }, data, }; @@ -177,8 +197,8 @@ export class AssetSearchService { * @param query The query to match with. */ async aggregate(query: AssetSearchQuery): Promise { - const ids = await this.searchIds(query); - const stats = await this.aggregateAssetIds(ids); + const [mapping, _total] = await this.searchAssetsByQuery(query); + const stats = await this.aggregateAssetIds([...mapping.keys()]); if (stats !== null) { return stats; } @@ -229,70 +249,78 @@ export class AssetSearchService { })); } - private async searchIds(query: AssetSearchQuery): Promise { - // Apply the query on Elasticsearch. - const queriedAssetIds = await this.searchAssetIdsByQuery(query); - - // Polygon searches need to be applied on the database. - // With no polygon being present, we can just use the ids returned by Elasticsearch. - return query.polygon == null ? queriedAssetIds : await this.filterAssetIdsByPolygon(query.polygon, queriedAssetIds); - } + private async searchAssetsByQuery( + query: AssetSearchQuery, + page: PageOptions = {} + ): Promise<[Map, number]> { + const BATCH_SIZE = 10_000; - private async searchAssetIdsByQuery(query: AssetSearchQuery): Promise { const elasticQuery = mapQueryToElasticDsl(query); - const matchedAssetIds: number[] = []; - for (;;) { + const matchedAssets = new Map(); + let lastAssetId: number | null = null; + let totalCount: number | null = null; + + let remainingOffset = page.offset ?? 0; + while (remainingOffset > 0) { + const remainingLimit = Math.min(BATCH_SIZE, remainingOffset); + if (remainingLimit <= 0) { + break; + } const response = await this.elastic.search({ index: INDEX, query: elasticQuery, fields: ['assetId'], - size: 10_000, + size: remainingLimit, sort: { assetId: 'desc', }, - track_total_hits: true, _source: false, - search_after: - matchedAssetIds.length === 0 ? undefined : ([matchedAssetIds[matchedAssetIds.length - 1]] as number[]), + track_total_hits: totalCount == null, + search_after: lastAssetId == null ? undefined : [lastAssetId], }); - matchedAssetIds.push(...response.hits.hits.map((hit) => hit.fields!['assetId'][0])); - const totalHits = (response.hits.total as SearchTotalHits).value; - if (totalHits <= matchedAssetIds.length) { - return matchedAssetIds; + totalCount ??= (response.hits.total as SearchTotalHits).value as number; + if (response.hits.hits.length < remainingLimit) { + return [matchedAssets, totalCount]; } + remainingOffset -= response.hits.hits.length; + lastAssetId = response.hits.hits[response.hits.hits.length - 1].fields!['assetId'][0] as number; } - } - private async filterAssetIdsByPolygon(polygon: Polygon, assetIds: number[]): Promise { - if (assetIds.length === 0) { - return []; - } - if (polygon.length === 0) { - return assetIds; + for (;;) { + const remainingLimit = page.limit == null ? BATCH_SIZE : Math.min(BATCH_SIZE, page.limit - matchedAssets.size); + if (remainingLimit <= 0 && totalCount != null) { + return [matchedAssets, totalCount]; + } + const response = await this.elastic.search({ + index: INDEX, + query: elasticQuery, + fields: ['assetId', 'data'], + size: remainingLimit, + sort: { + assetId: 'desc', + }, + track_total_hits: totalCount == null, + _source: false, + search_after: lastAssetId == null ? undefined : [lastAssetId], + }); + totalCount ??= (response.hits.total as SearchTotalHits).value as number; + if (response.hits.hits.length === 0) { + return [matchedAssets, totalCount]; + } + + for (const hit of response.hits.hits) { + const assetId: number = hit.fields!['assetId'][0]; + const data = hit.fields!['data'][0]; + matchedAssets.set(assetId, data); + lastAssetId = assetId; + } + if (totalCount <= (page.offset ?? 0) + matchedAssets.size) { + return [matchedAssets, totalCount]; + } } - const conditions: Prisma.Sql[] = [Prisma.sql`a.asset_id IN (${Prisma.join(assetIds, ',')})`]; - - const sqlPolygonParams = polygon.map(({ x, y }) => `${y} ${x}`).join(','); - const sqlPolygon = `polygon((${sqlPolygonParams}))`; - conditions.push(Prisma.sql` - ST_CONTAINS( - ST_GEOMFROMTEXT(${sqlPolygon}, 2056), - ST_CENTROID(s.geom) - ) - `); - const matchedAssets = await this.prisma.$queryRaw>` - SELECT DISTINCT a.asset_id as "assetId" - FROM public.asset a - LEFT JOIN - public.all_study s - ON - s.asset_id = a.asset_id - WHERE ${Prisma.join(conditions, ' AND ')} - `; - return matchedAssets.map((it) => it.assetId); } - private async aggregateAssetIds(assetIds: number[]): Promise { + private async aggregateAssetIds(assetIds: AssetId[]): Promise { if (assetIds.length === 0) { return null; } @@ -564,7 +592,7 @@ export class AssetSearchService { }; } - private async mapAssetToElastic(asset: AssetEditDetailFromPostgres): Promise { + private async mapAssetToElastic(asset: AssetEditDetail): Promise { const contacts = await this.prisma.contact.findMany({ select: { name: true, @@ -573,9 +601,11 @@ export class AssetSearchService { contactId: { in: asset.assetContacts.map((it) => it.contactId) }, }, }); - const geometryCodes = asset.studies - .map((study) => study.geomText.split('(', 2)[0]) - .map((prefix) => { + const geometryCodes: GeometryCode[] = []; + const studyLocations: ElasticPoint[] = []; + for (const study of asset.studies) { + const geometryCode = (() => { + const prefix = study.geomText.split('(', 2)[0]; switch (prefix) { case 'POINT': return GeometryCode.Point; @@ -586,7 +616,15 @@ export class AssetSearchService { default: throw new Error(`unknown geomText prefix: ${prefix}`); } - }); + })(); + geometryCodes.push(geometryCode); + + const fullStudy = await this.studyRepo.find(study.studyId as StudyId); + if (fullStudy != null) { + studyLocations.push(mapLv95ToElastic(fullStudy.center)); + } + } + const languageItemCodes = asset.assetLanguages.length === 0 ? ['None'] : asset.assetLanguages.map((it) => it.languageItemCode); return { @@ -602,10 +640,21 @@ export class AssetSearchService { contactNames: contacts.map((it) => it.name), manCatLabelItemCodes: asset.manCatLabelRefs, geometryCodes: geometryCodes.length > 0 ? [...new Set(geometryCodes)] : ['None'], + studyLocations, + data: JSON.stringify(AssetEditDetail.encode(asset)), }; } } +const lv95Projection = + '+proj=somerc +lat_0=46.95240555555556 +lon_0=7.439583333333333 +k_0=1 +x_0=2600000 +y_0=1200000 +ellps=bessel +towgs84=674.374,15.056,405.346,0,0,0,0 +units=m +no_defs'; +const wgs84Projection = proj4.WGS84; + +const mapLv95ToElastic = (lv95: LV95): ElasticPoint => { + const wgs = proj4(lv95Projection, wgs84Projection, [lv95.x as number, lv95.y as number]); + return { lat: wgs[0], lon: wgs[1] }; +}; + const mapQueryToElasticDsl = (query: AssetSearchQuery): QueryDslQueryContainer => { const scope = ['titlePublic', 'titleOriginal', 'contactNames', 'sgsId']; const queries: QueryDslQueryContainer[] = []; @@ -660,6 +709,15 @@ const mapQueryToElasticDsl = (query: AssetSearchQuery): QueryDslQueryContainer = if (query.geometryCodes != null) { filters.push(makeArrayFilter('geometryCodes', query.geometryCodes)); } + if (query.polygon != null) { + queries.push({ + geo_polygon: { + studyLocations: { + points: query.polygon.map(mapLv95ToElastic), + }, + }, + }); + } return { bool: { diff --git a/apps/server-asset-sg/src/features/studies/studies.controller.ts b/apps/server-asset-sg/src/features/studies/studies.controller.ts index 851d12de..8679c596 100644 --- a/apps/server-asset-sg/src/features/studies/studies.controller.ts +++ b/apps/server-asset-sg/src/features/studies/studies.controller.ts @@ -2,7 +2,7 @@ import { Readable } from 'stream'; import { Controller, Get, Res } from '@nestjs/common'; import { Response } from 'express'; import { RequireRole } from '@/core/decorators/require-role.decorator'; -import { Study } from '@/features/studies/study.model'; +import { serializeStudyAsCsv, Study } from '@/features/studies/study.model'; import { StudyRepo } from '@/features/studies/study.repo'; import { Role } from '@/features/users/user.model'; @@ -61,7 +61,8 @@ export class StudiesController { // Write the current batch to the response. for (const study of studies) { - yield `${study.id.substring(6)};${study.assetId};${+study.isPoint};${study.center.x};${study.center.y}\n`; + yield serializeStudyAsCsv(study); + yield '\n'; } } } diff --git a/apps/server-asset-sg/src/features/studies/study.model.spec.ts b/apps/server-asset-sg/src/features/studies/study.model.spec.ts new file mode 100644 index 00000000..9baf87ba --- /dev/null +++ b/apps/server-asset-sg/src/features/studies/study.model.spec.ts @@ -0,0 +1,36 @@ +import { LV95X, LV95Y } from '@asset-sg/shared'; +import { serializeStudyAsCsv, Study } from './study.model'; + +describe('serializeStudyAsCsv', () => { + it('serializes a point study in CSV format', () => { + // Given + const study: Study = { + id: `study_area_123`, + assetId: 2393, + isPoint: true, + center: { x: 2633499.729333331 as LV95X, y: 1171499.2243333303 as LV95Y }, + }; + + // When + const csv = serializeStudyAsCsv(study); + + // Then + expect(csv).toEqual('area_123;2393;1;2633499.729333331;1171499.2243333303'); + }); + + it('serializes a non-point study in CSV format', () => { + // Given + const study: Study = { + id: `study_location_934`, + assetId: 1549232, + isPoint: false, + center: { x: 2600230.056 as LV95X, y: 1198450.047 as LV95Y }, + }; + + // When + const csv = serializeStudyAsCsv(study); + + // Then + expect(csv).toEqual('location_934;1549232;0;2600230.056;1198450.047'); + }); +}); diff --git a/apps/server-asset-sg/src/features/studies/study.model.ts b/apps/server-asset-sg/src/features/studies/study.model.ts index 1b636695..7601edc4 100644 --- a/apps/server-asset-sg/src/features/studies/study.model.ts +++ b/apps/server-asset-sg/src/features/studies/study.model.ts @@ -14,3 +14,7 @@ export enum StudyType { Location = 'location', Trace = 'trace', } + +export const serializeStudyAsCsv = (study: Study): string => { + return `${study.id.substring(6)};${study.assetId};${+study.isPoint};${study.center.x};${study.center.y}`; +}; diff --git a/apps/server-asset-sg/src/features/studies/study.repo.ts b/apps/server-asset-sg/src/features/studies/study.repo.ts index 26899860..e2f8e021 100644 --- a/apps/server-asset-sg/src/features/studies/study.repo.ts +++ b/apps/server-asset-sg/src/features/studies/study.repo.ts @@ -1,4 +1,4 @@ -import { LV95, LV95FromSpaceSeparatedString } from '@asset-sg/shared'; +import { LV95, LV95FromSpaceSeparatedString, parseLV95 } from '@asset-sg/shared'; import { Injectable } from '@nestjs/common'; import { Prisma } from '@prisma/client'; import * as E from 'fp-ts/Either'; @@ -54,10 +54,9 @@ export class StudyRepo implements ReadRepo { ${condition} `; return studies.map((study) => { - const center = LV95FromSpaceSeparatedString.decode(study.center); return { ...study, - center: (center as E.Right).right, + center: parseLV95(study.center, { separator: ' ' }), }; }); } diff --git a/development/init/elasticsearch/mappings/swissgeol_asset_asset.json b/development/init/elasticsearch/mappings/swissgeol_asset_asset.json index 2c794a57..b4810b82 100644 --- a/development/init/elasticsearch/mappings/swissgeol_asset_asset.json +++ b/development/init/elasticsearch/mappings/swissgeol_asset_asset.json @@ -56,6 +56,9 @@ }, "usageCode": { "type": "keyword" + }, + "studyLocations": { + "type": "geo_point" } } } diff --git a/libs/asset-viewer/src/lib/services/asset-search.service.ts b/libs/asset-viewer/src/lib/services/asset-search.service.ts index 46757e56..4674349d 100644 --- a/libs/asset-viewer/src/lib/services/asset-search.service.ts +++ b/libs/asset-viewer/src/lib/services/asset-search.service.ts @@ -16,7 +16,7 @@ import { Observable, map, tap } from 'rxjs'; @Injectable({ providedIn: 'root' }) export class AssetSearchService { - constructor(private _httpClient: HttpClient, private store: Store) {} + constructor(private _httpClient: HttpClient) {} public search(searchQuery: AssetSearchQuery): Observable { return this._httpClient.post('/api/assets/search?limit=10000', searchQuery).pipe( diff --git a/libs/shared/src/lib/models/all-study.ts b/libs/shared/src/lib/models/all-study.ts index 60377213..f2943bfa 100644 --- a/libs/shared/src/lib/models/all-study.ts +++ b/libs/shared/src/lib/models/all-study.ts @@ -9,11 +9,3 @@ export const AllStudyDTOFromAPI = D.struct({ centroid: LV95FromSpaceSeparatedString, }); export type AllStudyDTOFromAPI = D.TypeOf; -export const AllStudyDTOsFromAPI = D.array(AllStudyDTOFromAPI); - -export interface AllStudyRaw { - studyId: string; - assetId: number; - isPoint: boolean; - centroid: string; -} diff --git a/libs/shared/src/lib/models/elastic-search-asset.ts b/libs/shared/src/lib/models/elastic-search-asset.ts index 9e0a9ba4..639ff0d9 100644 --- a/libs/shared/src/lib/models/elastic-search-asset.ts +++ b/libs/shared/src/lib/models/elastic-search-asset.ts @@ -16,4 +16,13 @@ export interface ElasticSearchAsset { contactNames: string[]; manCatLabelItemCodes: string[]; geometryCodes: GeometryCode[] | ['None']; + studyLocations: ElasticPoint[]; + data: SerializedAssetEditDetail; } + +export interface ElasticPoint { + lat: number; + lon: number; +} + +export type SerializedAssetEditDetail = string; diff --git a/libs/shared/src/lib/models/lv95.ts b/libs/shared/src/lib/models/lv95.ts index 8c0a0fb9..d05e8aa3 100644 --- a/libs/shared/src/lib/models/lv95.ts +++ b/libs/shared/src/lib/models/lv95.ts @@ -84,3 +84,11 @@ export const lv95RoundedToMillimeter = (lv95: LV95): LV95 => ({ x: roundToMillimeter(lv95.x) as LV95X, y: roundToMillimeter(lv95.y) as LV95Y, }); + +export const parseLV95 = (value: string, { separator }: { separator: string }): LV95 => { + const [x, y] = value.split(separator, 2); + return { + x: parseInt(x) as LV95X, + y: parseInt(y) as LV95Y, + }; +}; diff --git a/package-lock.json b/package-lock.json index 2d71550a..2a521e07 100644 --- a/package-lock.json +++ b/package-lock.json @@ -63,6 +63,7 @@ "node-fetch": "^3.3.1", "ol": "^7.2.2", "prisma": "^4.12.0", + "proj4": "^2.11.0", "query-string": "^8.1.0", "reflect-metadata": "^0.1.13", "rxjs": "7.8.1", @@ -107,6 +108,7 @@ "@types/jwk-to-pem": "^2.0.3", "@types/multer": "^1.4.7", "@types/node": "^18.16.9", + "@types/proj4": "^2.5.5", "@types/validator": "^13.7.10", "@typescript-eslint/eslint-plugin": "^7.10.0", "@typescript-eslint/parser": "7.3.0", @@ -11966,6 +11968,13 @@ "resolved": "https://registry.npmjs.org/@types/parse-json/-/parse-json-4.0.2.tgz", "integrity": "sha512-dISoDXWWQwUquiKsyZ4Ng+HX2KsPL7LyHKHQwgGFEA3IaKac4Obd+h2a/a6waisAoepJlBcx9paWqjA8/HVjCw==" }, + "node_modules/@types/proj4": { + "version": "2.5.5", + "resolved": "https://registry.npmjs.org/@types/proj4/-/proj4-2.5.5.tgz", + "integrity": "sha512-y4tHUVVoMEOm2nxRLQ2/ET8upj/pBmoutGxFw2LZJTQWPgWXI+cbxVEUFFmIzr/bpFR83hGDOTSXX6HBeObvZA==", + "dev": true, + "license": "MIT" + }, "node_modules/@types/qs": { "version": "6.9.15", "resolved": "https://registry.npmjs.org/@types/qs/-/qs-6.9.15.tgz", @@ -23693,6 +23702,12 @@ "node": ">= 0.6" } }, + "node_modules/mgrs": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/mgrs/-/mgrs-1.0.0.tgz", + "integrity": "sha512-awNbTOqCxK1DBGjalK3xqWIstBZgN6fxsMSiXLs9/spqWkF2pAhb2rrYCFSsr1/tT7PhcDGjZndG8SWYn0byYA==", + "license": "MIT" + }, "node_modules/micromatch": { "version": "4.0.7", "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.7.tgz", @@ -27280,6 +27295,16 @@ "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==" }, + "node_modules/proj4": { + "version": "2.11.0", + "resolved": "https://registry.npmjs.org/proj4/-/proj4-2.11.0.tgz", + "integrity": "sha512-SasuTkAx8HnWQHfIyhkdUNJorSJqINHAN3EyMWYiQRVorftz9DHz650YraFgczwgtHOxqnfuDxSNv3C8MUnHeg==", + "license": "MIT", + "dependencies": { + "mgrs": "1.0.0", + "wkt-parser": "^1.3.3" + } + }, "node_modules/promise-coalesce": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/promise-coalesce/-/promise-coalesce-1.1.2.tgz", @@ -31782,6 +31807,12 @@ "resolved": "https://registry.npmjs.org/wildcard/-/wildcard-2.0.1.tgz", "integrity": "sha512-CC1bOL87PIWSBhDcTrdeLo6eGT7mCFtrg0uIJtqJUFyK+eJnzl8A1niH56uu7KMa5XFrtiV+AQuHO3n7DsHnLQ==" }, + "node_modules/wkt-parser": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/wkt-parser/-/wkt-parser-1.3.3.tgz", + "integrity": "sha512-ZnV3yH8/k58ZPACOXeiHaMuXIiaTk1t0hSUVisbO0t4RjA5wPpUytcxeyiN2h+LZRrmuHIh/1UlrR9e7DHDvTw==", + "license": "MIT" + }, "node_modules/word-wrap": { "version": "1.2.5", "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.5.tgz", diff --git a/package.json b/package.json index 94e18aa4..3f89d24c 100644 --- a/package.json +++ b/package.json @@ -7,7 +7,7 @@ "start": "concurrently \"npm run start:server\" \"npm run start:client\"", "start:server": "npx nx serve server-asset-sg", "start:client": "npx nx serve client-asset-sg --disableHostCheck=true", - "build": "concurrently \"npm run build:server\" \"npm run build:client\"", + "build": "npx nx run-many -t build -p server-asset-sg client-asset-sg --configuration=production", "build:server": "npx nx build server-asset-sg --configuration=production", "build:client": "npx nx build client-asset-sg --configuration=production", "prisma": "dotenv -e apps/server-asset-sg/.env.local -- npx prisma", @@ -74,6 +74,7 @@ "node-fetch": "^3.3.1", "ol": "^7.2.2", "prisma": "^4.12.0", + "proj4": "^2.11.0", "query-string": "^8.1.0", "reflect-metadata": "^0.1.13", "rxjs": "7.8.1", @@ -97,6 +98,8 @@ "@angular/cli": "~17.3.0", "@angular/compiler-cli": "17.3.10", "@angular/language-service": "17.3.10", + "@eslint/eslintrc": "^2.1.1", + "@eslint/js": "~8.57.0", "@faker-js/faker": "^8.4.1", "@nestjs/schematics": "10.0.1", "@nestjs/testing": "10.0.2", @@ -110,12 +113,15 @@ "@nx/vite": "19.2.1", "@nx/workspace": "19.2.1", "@schematics/angular": "17.3.8", + "@stylistic/eslint-plugin": "^2.1.0", "@types/jest": "^29.4.4", "@types/jsonwebtoken": "^9.0.1", "@types/jwk-to-pem": "^2.0.3", "@types/multer": "^1.4.7", "@types/node": "^18.16.9", + "@types/proj4": "^2.5.5", "@types/validator": "^13.7.10", + "@typescript-eslint/eslint-plugin": "^7.10.0", "@typescript-eslint/parser": "7.3.0", "@vitest/coverage-v8": "1.3.1", "@vitest/ui": "1.3.1", @@ -123,36 +129,32 @@ "concurrently": "^7.6.0", "cypress": "^13.12.0", "dotenv-cli": "^7.4.1", + "eslint": "^8.57.0", + "eslint-config-prettier": "^9.1.0", "eslint-plugin-cypress": "2.13.4", "eslint-plugin-import": "^2.26.0", + "husky": "^9.0.11", "jest": "^29.7.0", "jest-environment-jsdom": "^29.7.0", "jest-preset-angular": "14.0.4", "jsdom": "^24.0.0", + "lint-staged": "^15.2.4", "ng-packagr": "^17.3.0", "nx": "19.2.1", "postcss": "^8.4.5", "postcss-import": "~14.1.0", "postcss-preset-env": "~7.5.0", "postcss-url": "~10.1.3", + "prettier": "2.8.8", "prettier-plugin-prisma": "^4.8.0", "ts-jest": "29.1.3", "ts-node": "10.9.1", "type-zoo": "^3.4.1", - "@eslint/eslintrc": "^2.1.1", - "@eslint/js": "~8.57.0", - "@stylistic/eslint-plugin": "^2.1.0", - "@typescript-eslint/eslint-plugin": "^7.10.0", - "eslint": "^8.57.0", "typescript": "~5.4.5", "vite": "^5.2.11", "vite-plugin-dts": "^3.9.1", "vite-plugin-eslint": "^1.8.1", "vite-tsconfig-paths": "^4.3.2", - "eslint-config-prettier": "^9.1.0", - "husky": "^9.0.11", - "lint-staged": "^15.2.4", - "prettier": "2.8.8", "vitest": "1.3.1" }, "prisma": {