diff --git a/.prettierignore b/.prettierignore index 24c4e11b..8f8e33c6 100644 --- a/.prettierignore +++ b/.prettierignore @@ -7,3 +7,4 @@ /dist /node_modules /tmp +/.idea diff --git a/apps/server-asset-sg/src/features/asset-old/asset.service.ts b/apps/server-asset-sg/src/features/asset-old/asset.service.ts index 2adc4d41..ffbb93b4 100644 --- a/apps/server-asset-sg/src/features/asset-old/asset.service.ts +++ b/apps/server-asset-sg/src/features/asset-old/asset.service.ts @@ -1,25 +1,11 @@ -import { DT, decodeError, isNotNil, unknownToError, unknownToUnknownError } from '@asset-sg/core'; -import { - AssetSearchParams, - BaseAssetDetail, - DateId, - DateIdFromDate, - SearchAssetResult, - UsageCode, - makeUsageCode, - AllStudyDTOFromAPI, - AllStudyRaw, -} from '@asset-sg/shared'; +import { decodeError, isNotNil, unknownToError, unknownToUnknownError } from '@asset-sg/core'; +import { AssetSearchParams, BaseAssetDetail, SearchAssetResult } from '@asset-sg/shared'; import { Injectable } from '@nestjs/common'; import { sequenceS } from 'fp-ts/Apply'; import * as A from 'fp-ts/Array'; -import { contramap } from 'fp-ts/Eq'; -import { Lazy, flow, pipe } from 'fp-ts/function'; -import * as NEA from 'fp-ts/NonEmptyArray'; -import * as N from 'fp-ts/number'; +import { flow, Lazy, pipe } from 'fp-ts/function'; import * as O from 'fp-ts/Option'; import * as RR from 'fp-ts/ReadonlyRecord'; -import * as S from 'fp-ts/string'; import * as TE from 'fp-ts/TaskEither'; import * as C from 'io-ts/Codec'; import * as D from 'io-ts/Decoder'; @@ -79,114 +65,6 @@ export class AssetService { ); } - // findAssetsByPolygon(polygon: [number, number][]) { - // return findAssetsByPolygon(this.prismaService, polygon); - // } - - // findAssetsByPolygon(polygon: [number, number][]) { - // const polygonParam = polygon.map(point => `${point[0]} ${point[1]}`).join(','); - - // return pipe( - // TE.tryCatch( - // () => - // this.prismaService.$queryRawUnsafe(` - // select - // a.asset_id as "assetId", - // a.title_public as "titlePublic", - // a.create_date as "createDateId", - // a.asset_kind_item_code as "assetKindItemCode", - // a.asset_format_item_code as "assetFormatItemCode", - // mclr.man_cat_label_item_code as "manCatLabelItemCode", - // ac.contact_id as "contactId", - // ac.role as "contactRole", - // s.study_id as "studyId", - // s.geom_text as "studyGeomText" - // from all_study s - // inner join asset a - // on s.asset_id = a.asset_id - // left join - // asset_contact ac - // on ac.asset_id = a.asset_id - // left join - // man_cat_label_ref mclr - // on ac.asset_id = mclr.asset_id - // where - // st_intersects(geom, st_geomfromtext('polygon((${polygonParam}))', 2056)) - // order by - // a.asset_id - // `), - // unknownToError, - // ), - // TE.chainW( - // flow( - // DBResultList.decode, - // E.mapLeft(e => new Error(D.draw(e))), - // TE.fromEither, - // ), - // ), - // TE.map( - // flow( - // NEA.fromArray, - // O.map(assets => { - // const orderedDates: NEA.NonEmptyArray = pipe( - // assets, - // NEA.map(a => a.createDate), - // NEA.uniq(DateIdOrd), - // NEA.sort(DateIdOrd), - // ); - // return SearchAssetResult.encode({ - // _tag: 'SearchAssetResultNonEmpty', - // aggregations: { - // ranges: { createDate: { min: NEA.head(orderedDates), max: NEA.last(orderedDates) } }, - // buckets: { - // authorIds: pipe( - // assets, - // A.map(a => a.contacts.filter(c => c.role === 'author').map(c => c.id)), - // A.flatten, - // NEA.fromArray, - // O.map( - // flow( - // NEA.groupBy(a => String(a)), - // R.map(g => ({ key: NEA.head(g), count: g.length })), - // R.toArray, - // A.map(([, value]) => value), - // ), - // ), - // O.getOrElseW(() => []), - // ), - // }, - // }, - // assets: assets.map(asset => ({ ...asset, score: 1 })), - // }); - // }), - // O.getOrElse(() => SearchAssetResult.encode({ _tag: 'SearchAssetResultEmpty' })), - // ), - // ), - // ); - // } - - // async findStudiesByPolygon(polygon: [number, number][]) { - // const polygonParam = polygon.map(point => `${point[0]} ${point[1]}`).join(','); - - // console.log( - // `select study_id as "studyId", geom_text as "geomText", asset_id as "assetId" from public.all_study where st_intersects(geom, st_geomfromtext('polygon((${polygonParam}))', 2056))`, - // ); - - // return await this.prismaService.$queryRawUnsafe( - // `select study_id as "studyId", geom_text as "geomText", asset_id as "assetId" - // from public.all_study - // where st_intersects(geom, st_geomfromtext('polygon((${polygonParam}))', 2056))`, - // ); - // } - - // async findStudiesByAssetId(assetId: number) { - // return await this.prismaService.$queryRawUnsafe( - // `select study_id as "studyId", geom_text as "geomText", asset_id as "assetId" - // from public.all_study - // where asset_id=${assetId}}`, - // ); - // } - getReferenceData() { const qt = (f: Lazy>, key: K, newKey: string) => pipe( @@ -276,83 +154,3 @@ export class AssetService { ); } } - -const DBResultRaw = D.struct({ - assetId: D.number, - titlePublic: D.string, - contactId: DT.optionFromNullable(D.number), - contactRole: DT.optionFromNullable(D.string), - createDate: DateIdFromDate, - assetKindItemCode: D.string, - assetFormatItemCode: D.string, - languageItemCode: D.string, - manCatLabelItemCode: DT.optionFromNullable(D.string), - internalUse: D.boolean, - publicUse: D.boolean, - studyId: DT.optionFromNullable(D.string), - studyGeomText: DT.optionFromNullable(D.string), -}); -type DBResultRaw = D.TypeOf; - -// eslint-disable-next-line @typescript-eslint/no-unused-vars -const DBResultRawList = D.array(DBResultRaw); - -type DBResultRawList = D.TypeOf; - -type DBResult = { - assetId: number; - titlePublic: string; - createDate: DateId; - assetKindItemCode: string; - assetFormatItemCode: string; - languageItemCode: string; - manCatLabelItemCodes: Array; - usageCode: UsageCode; - contacts: Array<{ role: string; id: number }>; - studies: Array<{ studyId: string; geomText: string }>; -}; - -const eqStudyByStudyId = contramap((x: { studyId: string; geomText: string }) => x.studyId)(S.Eq); -const eqContactByContactId = contramap((x: { contactId: number; contactRole: string }) => x.contactId)(N.Eq); - -// eslint-disable-next-line @typescript-eslint/no-unused-vars -const dbResultRawToDBResult = (dbResultRawList: NEA.NonEmptyArray): DBResult => { - const { - assetId, - titlePublic, - createDate, - assetFormatItemCode, - assetKindItemCode, - languageItemCode, - internalUse, - publicUse, - } = NEA.head(dbResultRawList); - return { - assetId, - titlePublic, - createDate, - assetFormatItemCode, - assetKindItemCode, - languageItemCode, - contacts: pipe( - dbResultRawList, - NEA.map((x) => sequenceS(O.Apply)({ contactId: x.contactId, contactRole: x.contactRole })), - A.compact, - A.uniq(eqContactByContactId), - A.map((a) => ({ id: a.contactId, role: a.contactRole })) - ), - studies: pipe( - dbResultRawList, - NEA.map((x) => sequenceS(O.Apply)({ studyId: x.studyId, geomText: x.studyGeomText })), - A.compact, - A.uniq(eqStudyByStudyId) - ), - manCatLabelItemCodes: pipe( - dbResultRawList, - NEA.map((x) => x.manCatLabelItemCode), - A.compact, - A.uniq(S.Eq) - ), - usageCode: makeUsageCode(publicUse, internalUse), - }; -}; diff --git a/apps/server-asset-sg/src/features/assets/search/asset-search.controller.ts b/apps/server-asset-sg/src/features/assets/search/asset-search.controller.ts index efc10a13..6d21584e 100644 --- a/apps/server-asset-sg/src/features/assets/search/asset-search.controller.ts +++ b/apps/server-asset-sg/src/features/assets/search/asset-search.controller.ts @@ -1,14 +1,12 @@ import { - AssetEditDetail, AssetSearchQueryDTO, AssetSearchResult, AssetSearchResultDTO, AssetSearchStats, AssetSearchStatsDTO, } from '@asset-sg/shared'; -import { Body, Controller, Post, Query, ValidationPipe } from '@nestjs/common'; +import { Body, Controller, HttpCode, HttpStatus, Post, Query, ValidationPipe } from '@nestjs/common'; import { plainToInstance } from 'class-transformer'; - import { AssetSearchService } from '@/features/assets/search/asset-search.service'; @Controller('/assets/search') @@ -16,6 +14,7 @@ export class AssetSearchController { constructor(private readonly assetSearchService: AssetSearchService) {} @Post('/') + @HttpCode(HttpStatus.OK) async search( @Body(new ValidationPipe({ transform: true, whitelist: true, forbidNonWhitelisted: true })) query: AssetSearchQueryDTO, @@ -28,14 +27,12 @@ export class AssetSearchController { ): Promise { limit = limit == null ? limit : Number(limit); offset = offset == null ? offset : Number(offset); - const result = await this.assetSearchService.search(query, { limit, offset }); - return plainToInstance(AssetSearchResultDTO, { - ...result, - data: result.data.map(AssetEditDetail.encode) as unknown as AssetEditDetail[], - }); + const result = await this.assetSearchService.search(query, { limit, offset, decode: false }); + return plainToInstance(AssetSearchResultDTO, result); } @Post('/stats') + @HttpCode(HttpStatus.OK) async showStats( @Body(new ValidationPipe({ transform: true, whitelist: true, forbidNonWhitelisted: true })) query: AssetSearchQueryDTO diff --git a/apps/server-asset-sg/src/features/assets/search/asset-search.http b/apps/server-asset-sg/src/features/assets/search/asset-search.http index 8f0daa16..37df4b73 100644 --- a/apps/server-asset-sg/src/features/assets/search/asset-search.http +++ b/apps/server-asset-sg/src/features/assets/search/asset-search.http @@ -1,18 +1,10 @@ -### Start asset sync -POST {{host}}/api/assets/sync -Authorization: Impersonate {{user}} - -### Show asset sync progress -GET {{host}}/api/assets/sync -Authorization: Impersonate {{user}} - ### Search assets -POST {{host}}/api/assets/search?limit=1 +POST {{host}}/api/assets/search?limit=10000 Content-Type: application/json Authorization: Impersonate {{user}} { - "languageItemCodes": ["DE"] + "text": "n1" } ### Show assets search stats diff --git a/apps/server-asset-sg/src/features/assets/search/asset-search.service.spec.ts b/apps/server-asset-sg/src/features/assets/search/asset-search.service.spec.ts index ce5a99dc..d5738e2f 100644 --- a/apps/server-asset-sg/src/features/assets/search/asset-search.service.spec.ts +++ b/apps/server-asset-sg/src/features/assets/search/asset-search.service.spec.ts @@ -32,12 +32,14 @@ import { PrismaService } from '@/core/prisma.service'; import { fakeAssetPatch, fakeAssetUsage, fakeContact, fakeUser } from '@/features/asset-old/asset-edit.fake'; import { AssetData, AssetEditRepo } from '@/features/asset-old/asset-edit.repo'; import { AssetEditDetail } from '@/features/asset-old/asset-edit.service'; +import { StudyRepo } from '@/features/studies/study.repo'; describe(AssetSearchService, () => { const elastic = openElasticsearchClient(); const prisma = new PrismaService(); const assetRepo = new AssetEditRepo(prisma); - const service = new AssetSearchService(elastic, prisma, assetRepo); + const studyRepo = new StudyRepo(prisma); + const service = new AssetSearchService(elastic, prisma, assetRepo, studyRepo); beforeAll(async () => { const existsIndex = await elastic.indices.exists({ index: ASSET_ELASTIC_INDEX }); diff --git a/apps/server-asset-sg/src/features/assets/search/asset-search.service.ts b/apps/server-asset-sg/src/features/assets/search/asset-search.service.ts index 694cb19b..35fc35b8 100644 --- a/apps/server-asset-sg/src/features/assets/search/asset-search.service.ts +++ b/apps/server-asset-sg/src/features/assets/search/asset-search.service.ts @@ -1,20 +1,23 @@ /* eslint-disable @typescript-eslint/no-non-null-assertion */ import { AssetByTitle, + AssetEditDetail, AssetSearchQuery, AssetSearchResult, AssetSearchStats, + dateFromDateId, DateId, + dateIdFromDate, + ElasticPoint, ElasticSearchAsset, GeometryCode, - Polygon, + LV95, + makeUsageCode, SearchAssetAggregations, SearchAssetResult, + SerializedAssetEditDetail, UsageCode, ValueCount, - dateFromDateId, - dateIdFromDate, - makeUsageCode, } from '@asset-sg/shared'; import { Client as ElasticsearchClient } from '@elastic/elasticsearch'; import { @@ -24,26 +27,28 @@ import { SearchTotalHits, } from '@elastic/elasticsearch/lib/api/types'; import { Injectable } from '@nestjs/common'; -import { Prisma } from '@prisma/client'; +import * as E from 'fp-ts/Either'; +import proj4 from 'proj4'; // eslint-disable-next-line @nx/enforce-module-boundaries import indexMapping from '../../../../../../development/init/elasticsearch/mappings/swissgeol_asset_asset.json'; +import { AssetId } from '../asset.model'; import { PrismaService } from '@/core/prisma.service'; import { AssetEditRepo } from '@/features/asset-old/asset-edit.repo'; -import { AssetEditDetail } from '@/features/asset-old/asset-edit.service'; -import { AssetEditDetailFromPostgres } from '@/models/asset-edit-detail'; +import { StudyId } from '@/features/studies/study.model'; +import { StudyRepo } from '@/features/studies/study.repo'; const INDEX = 'swissgeol_asset_asset'; export { INDEX as ASSET_ELASTIC_INDEX }; interface SearchOptions { scope: Array; - assetIds?: number[]; + assetIds?: AssetId[]; } interface ElasticSearchResult { - scoresByAssetId: Map; + scoresByAssetId: Map; aggs: SearchAssetAggregations; } @@ -52,7 +57,8 @@ export class AssetSearchService { constructor( private readonly elastic: ElasticsearchClient, private readonly prisma: PrismaService, - private readonly assetRepo: AssetEditRepo + private readonly assetRepo: AssetEditRepo, + private readonly studyRepo: StudyRepo ) {} register(oneOrMore: AssetEditDetail | AssetEditDetail[]): Promise { @@ -105,7 +111,7 @@ export class AssetSearchService { ...indexMapping, }); - // Refresh the sync index so we can reindex its contents. + // Refresh the sync index, so we can reindex its contents. await this.elastic.indices.refresh({ index: SYNC_INDEX }); // Copy the sync index's contents into the empty asset index. @@ -126,11 +132,16 @@ export class AssetSearchService { { index, shouldRefresh = false }: { index: string; shouldRefresh?: boolean } ): Promise { const assets = Array.isArray(oneOrMore) ? oneOrMore : [oneOrMore]; - const elasticAssets = await Promise.all(assets.map((asset) => this.mapAssetToElastic(asset))); - const operations = elasticAssets.reduce((ops, elasticAsset) => { - ops.push({ index: { _index: index, _id: `${elasticAsset.assetId}` } }, elasticAsset); - return ops; - }, [] as Array); + const operations: Array = Array(assets.length * 2); + const mappings: Array> = Array(assets.length); + for (let i = 0; i < assets.length; i++) { + const asset = assets[i]; + mappings[i] = this.mapAssetToElastic(asset).then((elasticAsset) => { + operations[i * 2] = { index: { _index: index, _id: `${elasticAsset.assetId}` } }; + operations[i * 2 + 1] = elasticAsset; + }); + } + await Promise.all(mappings); await this.elastic.bulk({ index, refresh: shouldRefresh, @@ -152,20 +163,29 @@ export class AssetSearchService { * @param query The query to match with. * @param limit The maximum amount of assets to load. Defaults to `100`. * @param offset The amount of assets being skipped before loading the assets. + * @param decode Whether to decode the assets. If this is set to `false`, the assets should be decoded via `AssetEditDetail` before accessing them. + * This option primarily exists so that the assets are not decoded and directly re-encoded when returning them via API. */ - async search(query: AssetSearchQuery, { limit = 100, offset = 0 }: PageOptions = {}): Promise { + async search( + query: AssetSearchQuery, + { limit = 100, offset = 0, decode = true }: PageOptions & { decode?: boolean } = {} + ): Promise { // Apply the query to find all matching ids. - const ids = await this.searchIds(query); + const [serializedAssets, total] = await this.searchAssetsByQuery(query, { limit, offset }); // Load the matched assets from the database. - const data = await this.assetRepo.list({ ids, limit, offset }); + const data: AssetEditDetail[] = []; + for (const serializedAsset of serializedAssets.values()) { + const asset = JSON.parse(serializedAsset); + data.push(decode ? (AssetEditDetail.decode(asset) as E.Right).right : asset); + } // Return the matched data in a paginated format. return { page: { offset, size: data.length, - total: ids.length, + total, }, data, }; @@ -177,8 +197,8 @@ export class AssetSearchService { * @param query The query to match with. */ async aggregate(query: AssetSearchQuery): Promise { - const ids = await this.searchIds(query); - const stats = await this.aggregateAssetIds(ids); + const [mapping, _total] = await this.searchAssetsByQuery(query); + const stats = await this.aggregateAssetIds([...mapping.keys()]); if (stats !== null) { return stats; } @@ -229,70 +249,78 @@ export class AssetSearchService { })); } - private async searchIds(query: AssetSearchQuery): Promise { - // Apply the query on Elasticsearch. - const queriedAssetIds = await this.searchAssetIdsByQuery(query); - - // Polygon searches need to be applied on the database. - // With no polygon being present, we can just use the ids returned by Elasticsearch. - return query.polygon == null ? queriedAssetIds : await this.filterAssetIdsByPolygon(query.polygon, queriedAssetIds); - } + private async searchAssetsByQuery( + query: AssetSearchQuery, + page: PageOptions = {} + ): Promise<[Map, number]> { + const BATCH_SIZE = 10_000; - private async searchAssetIdsByQuery(query: AssetSearchQuery): Promise { const elasticQuery = mapQueryToElasticDsl(query); - const matchedAssetIds: number[] = []; - for (;;) { + const matchedAssets = new Map(); + let lastAssetId: number | null = null; + let totalCount: number | null = null; + + let remainingOffset = page.offset ?? 0; + while (remainingOffset > 0) { + const remainingLimit = Math.min(BATCH_SIZE, remainingOffset); + if (remainingLimit <= 0) { + break; + } const response = await this.elastic.search({ index: INDEX, query: elasticQuery, fields: ['assetId'], - size: 10_000, + size: remainingLimit, sort: { assetId: 'desc', }, - track_total_hits: true, _source: false, - search_after: - matchedAssetIds.length === 0 ? undefined : ([matchedAssetIds[matchedAssetIds.length - 1]] as number[]), + track_total_hits: totalCount == null, + search_after: lastAssetId == null ? undefined : [lastAssetId], }); - matchedAssetIds.push(...response.hits.hits.map((hit) => hit.fields!['assetId'][0])); - const totalHits = (response.hits.total as SearchTotalHits).value; - if (totalHits <= matchedAssetIds.length) { - return matchedAssetIds; + totalCount ??= (response.hits.total as SearchTotalHits).value as number; + if (response.hits.hits.length < remainingLimit) { + return [matchedAssets, totalCount]; } + remainingOffset -= response.hits.hits.length; + lastAssetId = response.hits.hits[response.hits.hits.length - 1].fields!['assetId'][0] as number; } - } - private async filterAssetIdsByPolygon(polygon: Polygon, assetIds: number[]): Promise { - if (assetIds.length === 0) { - return []; - } - if (polygon.length === 0) { - return assetIds; + for (;;) { + const remainingLimit = page.limit == null ? BATCH_SIZE : Math.min(BATCH_SIZE, page.limit - matchedAssets.size); + if (remainingLimit <= 0 && totalCount != null) { + return [matchedAssets, totalCount]; + } + const response = await this.elastic.search({ + index: INDEX, + query: elasticQuery, + fields: ['assetId', 'data'], + size: remainingLimit, + sort: { + assetId: 'desc', + }, + track_total_hits: totalCount == null, + _source: false, + search_after: lastAssetId == null ? undefined : [lastAssetId], + }); + totalCount ??= (response.hits.total as SearchTotalHits).value as number; + if (response.hits.hits.length === 0) { + return [matchedAssets, totalCount]; + } + + for (const hit of response.hits.hits) { + const assetId: number = hit.fields!['assetId'][0]; + const data = hit.fields!['data'][0]; + matchedAssets.set(assetId, data); + lastAssetId = assetId; + } + if (totalCount <= (page.offset ?? 0) + matchedAssets.size) { + return [matchedAssets, totalCount]; + } } - const conditions: Prisma.Sql[] = [Prisma.sql`a.asset_id IN (${Prisma.join(assetIds, ',')})`]; - - const sqlPolygonParams = polygon.map(({ x, y }) => `${y} ${x}`).join(','); - const sqlPolygon = `polygon((${sqlPolygonParams}))`; - conditions.push(Prisma.sql` - ST_CONTAINS( - ST_GEOMFROMTEXT(${sqlPolygon}, 2056), - ST_CENTROID(s.geom) - ) - `); - const matchedAssets = await this.prisma.$queryRaw>` - SELECT DISTINCT a.asset_id as "assetId" - FROM public.asset a - LEFT JOIN - public.all_study s - ON - s.asset_id = a.asset_id - WHERE ${Prisma.join(conditions, ' AND ')} - `; - return matchedAssets.map((it) => it.assetId); } - private async aggregateAssetIds(assetIds: number[]): Promise { + private async aggregateAssetIds(assetIds: AssetId[]): Promise { if (assetIds.length === 0) { return null; } @@ -564,7 +592,7 @@ export class AssetSearchService { }; } - private async mapAssetToElastic(asset: AssetEditDetailFromPostgres): Promise { + private async mapAssetToElastic(asset: AssetEditDetail): Promise { const contacts = await this.prisma.contact.findMany({ select: { name: true, @@ -573,9 +601,11 @@ export class AssetSearchService { contactId: { in: asset.assetContacts.map((it) => it.contactId) }, }, }); - const geometryCodes = asset.studies - .map((study) => study.geomText.split('(', 2)[0]) - .map((prefix) => { + const geometryCodes: GeometryCode[] = []; + const studyLocations: ElasticPoint[] = []; + for (const study of asset.studies) { + const geometryCode = (() => { + const prefix = study.geomText.split('(', 2)[0]; switch (prefix) { case 'POINT': return GeometryCode.Point; @@ -586,7 +616,15 @@ export class AssetSearchService { default: throw new Error(`unknown geomText prefix: ${prefix}`); } - }); + })(); + geometryCodes.push(geometryCode); + + const fullStudy = await this.studyRepo.find(study.studyId as StudyId); + if (fullStudy != null) { + studyLocations.push(mapLv95ToElastic(fullStudy.center)); + } + } + const languageItemCodes = asset.assetLanguages.length === 0 ? ['None'] : asset.assetLanguages.map((it) => it.languageItemCode); return { @@ -602,10 +640,21 @@ export class AssetSearchService { contactNames: contacts.map((it) => it.name), manCatLabelItemCodes: asset.manCatLabelRefs, geometryCodes: geometryCodes.length > 0 ? [...new Set(geometryCodes)] : ['None'], + studyLocations, + data: JSON.stringify(AssetEditDetail.encode(asset)), }; } } +const lv95Projection = + '+proj=somerc +lat_0=46.95240555555556 +lon_0=7.439583333333333 +k_0=1 +x_0=2600000 +y_0=1200000 +ellps=bessel +towgs84=674.374,15.056,405.346,0,0,0,0 +units=m +no_defs'; +const wgs84Projection = proj4.WGS84; + +const mapLv95ToElastic = (lv95: LV95): ElasticPoint => { + const wgs = proj4(lv95Projection, wgs84Projection, [lv95.x as number, lv95.y as number]); + return { lat: wgs[0], lon: wgs[1] }; +}; + const mapQueryToElasticDsl = (query: AssetSearchQuery): QueryDslQueryContainer => { const scope = ['titlePublic', 'titleOriginal', 'contactNames', 'sgsId']; const queries: QueryDslQueryContainer[] = []; @@ -660,6 +709,15 @@ const mapQueryToElasticDsl = (query: AssetSearchQuery): QueryDslQueryContainer = if (query.geometryCodes != null) { filters.push(makeArrayFilter('geometryCodes', query.geometryCodes)); } + if (query.polygon != null) { + queries.push({ + geo_polygon: { + studyLocations: { + points: query.polygon.map(mapLv95ToElastic), + }, + }, + }); + } return { bool: { diff --git a/apps/server-asset-sg/src/features/studies/studies.controller.ts b/apps/server-asset-sg/src/features/studies/studies.controller.ts index 851d12de..8679c596 100644 --- a/apps/server-asset-sg/src/features/studies/studies.controller.ts +++ b/apps/server-asset-sg/src/features/studies/studies.controller.ts @@ -2,7 +2,7 @@ import { Readable } from 'stream'; import { Controller, Get, Res } from '@nestjs/common'; import { Response } from 'express'; import { RequireRole } from '@/core/decorators/require-role.decorator'; -import { Study } from '@/features/studies/study.model'; +import { serializeStudyAsCsv, Study } from '@/features/studies/study.model'; import { StudyRepo } from '@/features/studies/study.repo'; import { Role } from '@/features/users/user.model'; @@ -61,7 +61,8 @@ export class StudiesController { // Write the current batch to the response. for (const study of studies) { - yield `${study.id.substring(6)};${study.assetId};${+study.isPoint};${study.center.x};${study.center.y}\n`; + yield serializeStudyAsCsv(study); + yield '\n'; } } } diff --git a/apps/server-asset-sg/src/features/studies/study.model.spec.ts b/apps/server-asset-sg/src/features/studies/study.model.spec.ts new file mode 100644 index 00000000..9baf87ba --- /dev/null +++ b/apps/server-asset-sg/src/features/studies/study.model.spec.ts @@ -0,0 +1,36 @@ +import { LV95X, LV95Y } from '@asset-sg/shared'; +import { serializeStudyAsCsv, Study } from './study.model'; + +describe('serializeStudyAsCsv', () => { + it('serializes a point study in CSV format', () => { + // Given + const study: Study = { + id: `study_area_123`, + assetId: 2393, + isPoint: true, + center: { x: 2633499.729333331 as LV95X, y: 1171499.2243333303 as LV95Y }, + }; + + // When + const csv = serializeStudyAsCsv(study); + + // Then + expect(csv).toEqual('area_123;2393;1;2633499.729333331;1171499.2243333303'); + }); + + it('serializes a non-point study in CSV format', () => { + // Given + const study: Study = { + id: `study_location_934`, + assetId: 1549232, + isPoint: false, + center: { x: 2600230.056 as LV95X, y: 1198450.047 as LV95Y }, + }; + + // When + const csv = serializeStudyAsCsv(study); + + // Then + expect(csv).toEqual('location_934;1549232;0;2600230.056;1198450.047'); + }); +}); diff --git a/apps/server-asset-sg/src/features/studies/study.model.ts b/apps/server-asset-sg/src/features/studies/study.model.ts index 1b636695..7601edc4 100644 --- a/apps/server-asset-sg/src/features/studies/study.model.ts +++ b/apps/server-asset-sg/src/features/studies/study.model.ts @@ -14,3 +14,7 @@ export enum StudyType { Location = 'location', Trace = 'trace', } + +export const serializeStudyAsCsv = (study: Study): string => { + return `${study.id.substring(6)};${study.assetId};${+study.isPoint};${study.center.x};${study.center.y}`; +}; diff --git a/apps/server-asset-sg/src/features/studies/study.repo.ts b/apps/server-asset-sg/src/features/studies/study.repo.ts index 26899860..e2f8e021 100644 --- a/apps/server-asset-sg/src/features/studies/study.repo.ts +++ b/apps/server-asset-sg/src/features/studies/study.repo.ts @@ -1,4 +1,4 @@ -import { LV95, LV95FromSpaceSeparatedString } from '@asset-sg/shared'; +import { LV95, LV95FromSpaceSeparatedString, parseLV95 } from '@asset-sg/shared'; import { Injectable } from '@nestjs/common'; import { Prisma } from '@prisma/client'; import * as E from 'fp-ts/Either'; @@ -54,10 +54,9 @@ export class StudyRepo implements ReadRepo { ${condition} `; return studies.map((study) => { - const center = LV95FromSpaceSeparatedString.decode(study.center); return { ...study, - center: (center as E.Right).right, + center: parseLV95(study.center, { separator: ' ' }), }; }); } diff --git a/development/init/elasticsearch/mappings/swissgeol_asset_asset.json b/development/init/elasticsearch/mappings/swissgeol_asset_asset.json index 2c794a57..b4810b82 100644 --- a/development/init/elasticsearch/mappings/swissgeol_asset_asset.json +++ b/development/init/elasticsearch/mappings/swissgeol_asset_asset.json @@ -56,6 +56,9 @@ }, "usageCode": { "type": "keyword" + }, + "studyLocations": { + "type": "geo_point" } } } diff --git a/libs/asset-viewer/src/lib/services/asset-search.service.ts b/libs/asset-viewer/src/lib/services/asset-search.service.ts index 46757e56..4674349d 100644 --- a/libs/asset-viewer/src/lib/services/asset-search.service.ts +++ b/libs/asset-viewer/src/lib/services/asset-search.service.ts @@ -16,7 +16,7 @@ import { Observable, map, tap } from 'rxjs'; @Injectable({ providedIn: 'root' }) export class AssetSearchService { - constructor(private _httpClient: HttpClient, private store: Store) {} + constructor(private _httpClient: HttpClient) {} public search(searchQuery: AssetSearchQuery): Observable { return this._httpClient.post('/api/assets/search?limit=10000', searchQuery).pipe( diff --git a/libs/shared/src/lib/models/all-study.ts b/libs/shared/src/lib/models/all-study.ts index 60377213..f2943bfa 100644 --- a/libs/shared/src/lib/models/all-study.ts +++ b/libs/shared/src/lib/models/all-study.ts @@ -9,11 +9,3 @@ export const AllStudyDTOFromAPI = D.struct({ centroid: LV95FromSpaceSeparatedString, }); export type AllStudyDTOFromAPI = D.TypeOf; -export const AllStudyDTOsFromAPI = D.array(AllStudyDTOFromAPI); - -export interface AllStudyRaw { - studyId: string; - assetId: number; - isPoint: boolean; - centroid: string; -} diff --git a/libs/shared/src/lib/models/elastic-search-asset.ts b/libs/shared/src/lib/models/elastic-search-asset.ts index 9e0a9ba4..639ff0d9 100644 --- a/libs/shared/src/lib/models/elastic-search-asset.ts +++ b/libs/shared/src/lib/models/elastic-search-asset.ts @@ -16,4 +16,13 @@ export interface ElasticSearchAsset { contactNames: string[]; manCatLabelItemCodes: string[]; geometryCodes: GeometryCode[] | ['None']; + studyLocations: ElasticPoint[]; + data: SerializedAssetEditDetail; } + +export interface ElasticPoint { + lat: number; + lon: number; +} + +export type SerializedAssetEditDetail = string; diff --git a/libs/shared/src/lib/models/lv95.ts b/libs/shared/src/lib/models/lv95.ts index 8c0a0fb9..d05e8aa3 100644 --- a/libs/shared/src/lib/models/lv95.ts +++ b/libs/shared/src/lib/models/lv95.ts @@ -84,3 +84,11 @@ export const lv95RoundedToMillimeter = (lv95: LV95): LV95 => ({ x: roundToMillimeter(lv95.x) as LV95X, y: roundToMillimeter(lv95.y) as LV95Y, }); + +export const parseLV95 = (value: string, { separator }: { separator: string }): LV95 => { + const [x, y] = value.split(separator, 2); + return { + x: parseInt(x) as LV95X, + y: parseInt(y) as LV95Y, + }; +}; diff --git a/package-lock.json b/package-lock.json index 2d71550a..2a521e07 100644 --- a/package-lock.json +++ b/package-lock.json @@ -63,6 +63,7 @@ "node-fetch": "^3.3.1", "ol": "^7.2.2", "prisma": "^4.12.0", + "proj4": "^2.11.0", "query-string": "^8.1.0", "reflect-metadata": "^0.1.13", "rxjs": "7.8.1", @@ -107,6 +108,7 @@ "@types/jwk-to-pem": "^2.0.3", "@types/multer": "^1.4.7", "@types/node": "^18.16.9", + "@types/proj4": "^2.5.5", "@types/validator": "^13.7.10", "@typescript-eslint/eslint-plugin": "^7.10.0", "@typescript-eslint/parser": "7.3.0", @@ -11966,6 +11968,13 @@ "resolved": "https://registry.npmjs.org/@types/parse-json/-/parse-json-4.0.2.tgz", "integrity": "sha512-dISoDXWWQwUquiKsyZ4Ng+HX2KsPL7LyHKHQwgGFEA3IaKac4Obd+h2a/a6waisAoepJlBcx9paWqjA8/HVjCw==" }, + "node_modules/@types/proj4": { + "version": "2.5.5", + "resolved": "https://registry.npmjs.org/@types/proj4/-/proj4-2.5.5.tgz", + "integrity": "sha512-y4tHUVVoMEOm2nxRLQ2/ET8upj/pBmoutGxFw2LZJTQWPgWXI+cbxVEUFFmIzr/bpFR83hGDOTSXX6HBeObvZA==", + "dev": true, + "license": "MIT" + }, "node_modules/@types/qs": { "version": "6.9.15", "resolved": "https://registry.npmjs.org/@types/qs/-/qs-6.9.15.tgz", @@ -23693,6 +23702,12 @@ "node": ">= 0.6" } }, + "node_modules/mgrs": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/mgrs/-/mgrs-1.0.0.tgz", + "integrity": "sha512-awNbTOqCxK1DBGjalK3xqWIstBZgN6fxsMSiXLs9/spqWkF2pAhb2rrYCFSsr1/tT7PhcDGjZndG8SWYn0byYA==", + "license": "MIT" + }, "node_modules/micromatch": { "version": "4.0.7", "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.7.tgz", @@ -27280,6 +27295,16 @@ "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==" }, + "node_modules/proj4": { + "version": "2.11.0", + "resolved": "https://registry.npmjs.org/proj4/-/proj4-2.11.0.tgz", + "integrity": "sha512-SasuTkAx8HnWQHfIyhkdUNJorSJqINHAN3EyMWYiQRVorftz9DHz650YraFgczwgtHOxqnfuDxSNv3C8MUnHeg==", + "license": "MIT", + "dependencies": { + "mgrs": "1.0.0", + "wkt-parser": "^1.3.3" + } + }, "node_modules/promise-coalesce": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/promise-coalesce/-/promise-coalesce-1.1.2.tgz", @@ -31782,6 +31807,12 @@ "resolved": "https://registry.npmjs.org/wildcard/-/wildcard-2.0.1.tgz", "integrity": "sha512-CC1bOL87PIWSBhDcTrdeLo6eGT7mCFtrg0uIJtqJUFyK+eJnzl8A1niH56uu7KMa5XFrtiV+AQuHO3n7DsHnLQ==" }, + "node_modules/wkt-parser": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/wkt-parser/-/wkt-parser-1.3.3.tgz", + "integrity": "sha512-ZnV3yH8/k58ZPACOXeiHaMuXIiaTk1t0hSUVisbO0t4RjA5wPpUytcxeyiN2h+LZRrmuHIh/1UlrR9e7DHDvTw==", + "license": "MIT" + }, "node_modules/word-wrap": { "version": "1.2.5", "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.5.tgz", diff --git a/package.json b/package.json index 94e18aa4..3f89d24c 100644 --- a/package.json +++ b/package.json @@ -7,7 +7,7 @@ "start": "concurrently \"npm run start:server\" \"npm run start:client\"", "start:server": "npx nx serve server-asset-sg", "start:client": "npx nx serve client-asset-sg --disableHostCheck=true", - "build": "concurrently \"npm run build:server\" \"npm run build:client\"", + "build": "npx nx run-many -t build -p server-asset-sg client-asset-sg --configuration=production", "build:server": "npx nx build server-asset-sg --configuration=production", "build:client": "npx nx build client-asset-sg --configuration=production", "prisma": "dotenv -e apps/server-asset-sg/.env.local -- npx prisma", @@ -74,6 +74,7 @@ "node-fetch": "^3.3.1", "ol": "^7.2.2", "prisma": "^4.12.0", + "proj4": "^2.11.0", "query-string": "^8.1.0", "reflect-metadata": "^0.1.13", "rxjs": "7.8.1", @@ -97,6 +98,8 @@ "@angular/cli": "~17.3.0", "@angular/compiler-cli": "17.3.10", "@angular/language-service": "17.3.10", + "@eslint/eslintrc": "^2.1.1", + "@eslint/js": "~8.57.0", "@faker-js/faker": "^8.4.1", "@nestjs/schematics": "10.0.1", "@nestjs/testing": "10.0.2", @@ -110,12 +113,15 @@ "@nx/vite": "19.2.1", "@nx/workspace": "19.2.1", "@schematics/angular": "17.3.8", + "@stylistic/eslint-plugin": "^2.1.0", "@types/jest": "^29.4.4", "@types/jsonwebtoken": "^9.0.1", "@types/jwk-to-pem": "^2.0.3", "@types/multer": "^1.4.7", "@types/node": "^18.16.9", + "@types/proj4": "^2.5.5", "@types/validator": "^13.7.10", + "@typescript-eslint/eslint-plugin": "^7.10.0", "@typescript-eslint/parser": "7.3.0", "@vitest/coverage-v8": "1.3.1", "@vitest/ui": "1.3.1", @@ -123,36 +129,32 @@ "concurrently": "^7.6.0", "cypress": "^13.12.0", "dotenv-cli": "^7.4.1", + "eslint": "^8.57.0", + "eslint-config-prettier": "^9.1.0", "eslint-plugin-cypress": "2.13.4", "eslint-plugin-import": "^2.26.0", + "husky": "^9.0.11", "jest": "^29.7.0", "jest-environment-jsdom": "^29.7.0", "jest-preset-angular": "14.0.4", "jsdom": "^24.0.0", + "lint-staged": "^15.2.4", "ng-packagr": "^17.3.0", "nx": "19.2.1", "postcss": "^8.4.5", "postcss-import": "~14.1.0", "postcss-preset-env": "~7.5.0", "postcss-url": "~10.1.3", + "prettier": "2.8.8", "prettier-plugin-prisma": "^4.8.0", "ts-jest": "29.1.3", "ts-node": "10.9.1", "type-zoo": "^3.4.1", - "@eslint/eslintrc": "^2.1.1", - "@eslint/js": "~8.57.0", - "@stylistic/eslint-plugin": "^2.1.0", - "@typescript-eslint/eslint-plugin": "^7.10.0", - "eslint": "^8.57.0", "typescript": "~5.4.5", "vite": "^5.2.11", "vite-plugin-dts": "^3.9.1", "vite-plugin-eslint": "^1.8.1", "vite-tsconfig-paths": "^4.3.2", - "eslint-config-prettier": "^9.1.0", - "husky": "^9.0.11", - "lint-staged": "^15.2.4", - "prettier": "2.8.8", "vitest": "1.3.1" }, "prisma": {