Skip to content

Commit

Permalink
Move polygon search to Elasticsearch
Browse files Browse the repository at this point in the history 
Remove unused code

Replace fp-ts backed `LV95FromSpaceSeparatedString` with `parseLV95`

Extract and test `serializeStudyAsCsv`
  • Loading branch information
@daniel-va
daniel-va committed Jul 8, 2024
1 parent e3d18f7 commit 5bfbc69
Show file tree
Hide file tree
Showing 17 changed files with 255 additions and 322 deletions.
1 change: 1 addition & 0 deletions .prettierignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,3 +7,4 @@
/dist
/node_modules
/tmp
/.idea
208 changes: 3 additions & 205 deletions apps/server-asset-sg/src/features/asset-old/asset.service.ts
View file
Original file line number Diff line number Diff line change
@@ -1,25 +1,11 @@
import { DT, decodeError, isNotNil, unknownToError, unknownToUnknownError } from '@asset-sg/core';
import {
AssetSearchParams,
BaseAssetDetail,
DateId,
DateIdFromDate,
SearchAssetResult,
UsageCode,
makeUsageCode,
AllStudyDTOFromAPI,
AllStudyRaw,
} from '@asset-sg/shared';
import { decodeError, isNotNil, unknownToError, unknownToUnknownError } from '@asset-sg/core';
import { AssetSearchParams, BaseAssetDetail, SearchAssetResult } from '@asset-sg/shared';
import { Injectable } from '@nestjs/common';
import { sequenceS } from 'fp-ts/Apply';
import * as A from 'fp-ts/Array';
import { contramap } from 'fp-ts/Eq';
import { Lazy, flow, pipe } from 'fp-ts/function';
import * as NEA from 'fp-ts/NonEmptyArray';
import * as N from 'fp-ts/number';
import { flow, Lazy, pipe } from 'fp-ts/function';
import * as O from 'fp-ts/Option';
import * as RR from 'fp-ts/ReadonlyRecord';
import * as S from 'fp-ts/string';
import * as TE from 'fp-ts/TaskEither';
import * as C from 'io-ts/Codec';
import * as D from 'io-ts/Decoder';
Expand Down Expand Up @@ -79,114 +65,6 @@ export class AssetService {
);
}

// findAssetsByPolygon(polygon: [number, number][]) {
// return findAssetsByPolygon(this.prismaService, polygon);
// }

// findAssetsByPolygon(polygon: [number, number][]) {
// const polygonParam = polygon.map(point => `${point[0]} ${point[1]}`).join(',');

// return pipe(
// TE.tryCatch(
// () =>
// this.prismaService.$queryRawUnsafe(`
// select
// a.asset_id as "assetId",
// a.title_public as "titlePublic",
// a.create_date as "createDateId",
// a.asset_kind_item_code as "assetKindItemCode",
// a.asset_format_item_code as "assetFormatItemCode",
// mclr.man_cat_label_item_code as "manCatLabelItemCode",
// ac.contact_id as "contactId",
// ac.role as "contactRole",
// s.study_id as "studyId",
// s.geom_text as "studyGeomText"
// from all_study s
// inner join asset a
// on s.asset_id = a.asset_id
// left join
// asset_contact ac
// on ac.asset_id = a.asset_id
// left join
// man_cat_label_ref mclr
// on ac.asset_id = mclr.asset_id
// where
// st_intersects(geom, st_geomfromtext('polygon((${polygonParam}))', 2056))
// order by
// a.asset_id
// `),
// unknownToError,
// ),
// TE.chainW(
// flow(
// DBResultList.decode,
// E.mapLeft(e => new Error(D.draw(e))),
// TE.fromEither,
// ),
// ),
// TE.map(
// flow(
// NEA.fromArray,
// O.map(assets => {
// const orderedDates: NEA.NonEmptyArray<DateId> = pipe(
// assets,
// NEA.map(a => a.createDate),
// NEA.uniq(DateIdOrd),
// NEA.sort(DateIdOrd),
// );
// return SearchAssetResult.encode({
// _tag: 'SearchAssetResultNonEmpty',
// aggregations: {
// ranges: { createDate: { min: NEA.head(orderedDates), max: NEA.last(orderedDates) } },
// buckets: {
// authorIds: pipe(
// assets,
// A.map(a => a.contacts.filter(c => c.role === 'author').map(c => c.id)),
// A.flatten,
// NEA.fromArray,
// O.map(
// flow(
// NEA.groupBy(a => String(a)),
// R.map(g => ({ key: NEA.head(g), count: g.length })),
// R.toArray,
// A.map(([, value]) => value),
// ),
// ),
// O.getOrElseW(() => []),
// ),
// },
// },
// assets: assets.map(asset => ({ ...asset, score: 1 })),
// });
// }),
// O.getOrElse(() => SearchAssetResult.encode({ _tag: 'SearchAssetResultEmpty' })),
// ),
// ),
// );
// }

// async findStudiesByPolygon(polygon: [number, number][]) {
// const polygonParam = polygon.map(point => `${point[0]} ${point[1]}`).join(',');

// console.log(
// `select study_id as "studyId", geom_text as "geomText", asset_id as "assetId" from public.all_study where st_intersects(geom, st_geomfromtext('polygon((${polygonParam}))', 2056))`,
// );

// return await this.prismaService.$queryRawUnsafe(
// `select study_id as "studyId", geom_text as "geomText", asset_id as "assetId"
// from public.all_study
// where st_intersects(geom, st_geomfromtext('polygon((${polygonParam}))', 2056))`,
// );
// }

// async findStudiesByAssetId(assetId: number) {
// return await this.prismaService.$queryRawUnsafe(
// `select study_id as "studyId", geom_text as "geomText", asset_id as "assetId"
// from public.all_study
// where asset_id=${assetId}}`,
// );
// }

getReferenceData() {
const qt = <A, K extends keyof A>(f: Lazy<Promise<A[]>>, key: K, newKey: string) =>
pipe(
Expand Down Expand Up @@ -276,83 +154,3 @@ export class AssetService {
);
}
}

const DBResultRaw = D.struct({
assetId: D.number,
titlePublic: D.string,
contactId: DT.optionFromNullable(D.number),
contactRole: DT.optionFromNullable(D.string),
createDate: DateIdFromDate,
assetKindItemCode: D.string,
assetFormatItemCode: D.string,
languageItemCode: D.string,
manCatLabelItemCode: DT.optionFromNullable(D.string),
internalUse: D.boolean,
publicUse: D.boolean,
studyId: DT.optionFromNullable(D.string),
studyGeomText: DT.optionFromNullable(D.string),
});
type DBResultRaw = D.TypeOf<typeof DBResultRaw>;

// eslint-disable-next-line @typescript-eslint/no-unused-vars
const DBResultRawList = D.array(DBResultRaw);

type DBResultRawList = D.TypeOf<typeof DBResultRawList>;

type DBResult = {
assetId: number;
titlePublic: string;
createDate: DateId;
assetKindItemCode: string;
assetFormatItemCode: string;
languageItemCode: string;
manCatLabelItemCodes: Array<string>;
usageCode: UsageCode;
contacts: Array<{ role: string; id: number }>;
studies: Array<{ studyId: string; geomText: string }>;
};

const eqStudyByStudyId = contramap((x: { studyId: string; geomText: string }) => x.studyId)(S.Eq);
const eqContactByContactId = contramap((x: { contactId: number; contactRole: string }) => x.contactId)(N.Eq);

// eslint-disable-next-line @typescript-eslint/no-unused-vars
const dbResultRawToDBResult = (dbResultRawList: NEA.NonEmptyArray<DBResultRaw>): DBResult => {
const {
assetId,
titlePublic,
createDate,
assetFormatItemCode,
assetKindItemCode,
languageItemCode,
internalUse,
publicUse,
} = NEA.head(dbResultRawList);
return {
assetId,
titlePublic,
createDate,
assetFormatItemCode,
assetKindItemCode,
languageItemCode,
contacts: pipe(
dbResultRawList,
NEA.map((x) => sequenceS(O.Apply)({ contactId: x.contactId, contactRole: x.contactRole })),
A.compact,
A.uniq(eqContactByContactId),
A.map((a) => ({ id: a.contactId, role: a.contactRole }))
),
studies: pipe(
dbResultRawList,
NEA.map((x) => sequenceS(O.Apply)({ studyId: x.studyId, geomText: x.studyGeomText })),
A.compact,
A.uniq(eqStudyByStudyId)
),
manCatLabelItemCodes: pipe(
dbResultRawList,
NEA.map((x) => x.manCatLabelItemCode),
A.compact,
A.uniq(S.Eq)
),
usageCode: makeUsageCode(publicUse, internalUse),
};
};
13 changes: 5 additions & 8 deletions apps/server-asset-sg/src/features/assets/search/asset-search.controller.ts
View file
Original file line number Diff line number Diff line change
@@ -1,21 +1,20 @@
import {
AssetEditDetail,
AssetSearchQueryDTO,
AssetSearchResult,
AssetSearchResultDTO,
AssetSearchStats,
AssetSearchStatsDTO,
} from '@asset-sg/shared';
import { Body, Controller, Post, Query, ValidationPipe } from '@nestjs/common';
import { Body, Controller, HttpCode, HttpStatus, Post, Query, ValidationPipe } from '@nestjs/common';
import { plainToInstance } from 'class-transformer';

import { AssetSearchService } from '@/features/assets/search/asset-search.service';

@Controller('/assets/search')
export class AssetSearchController {
constructor(private readonly assetSearchService: AssetSearchService) {}

@Post('/')
@HttpCode(HttpStatus.OK)
async search(
@Body(new ValidationPipe({ transform: true, whitelist: true, forbidNonWhitelisted: true }))
query: AssetSearchQueryDTO,
Expand All @@ -28,14 +27,12 @@ export class AssetSearchController {
): Promise<AssetSearchResult> {
limit = limit == null ? limit : Number(limit);
offset = offset == null ? offset : Number(offset);
const result = await this.assetSearchService.search(query, { limit, offset });
return plainToInstance(AssetSearchResultDTO, {
...result,
data: result.data.map(AssetEditDetail.encode) as unknown as AssetEditDetail[],
});
const result = await this.assetSearchService.search(query, { limit, offset, decode: false });
return plainToInstance(AssetSearchResultDTO, result);
}

@Post('/stats')
@HttpCode(HttpStatus.OK)
async showStats(
@Body(new ValidationPipe({ transform: true, whitelist: true, forbidNonWhitelisted: true }))
query: AssetSearchQueryDTO
Expand Down
12 changes: 2 additions & 10 deletions apps/server-asset-sg/src/features/assets/search/asset-search.http
View file
Original file line number Diff line number Diff line change
@@ -1,18 +1,10 @@
### Start asset sync
POST {{host}}/api/assets/sync
Authorization: Impersonate {{user}}

### Show asset sync progress
GET {{host}}/api/assets/sync
Authorization: Impersonate {{user}}

### Search assets
POST {{host}}/api/assets/search?limit=1
POST {{host}}/api/assets/search?limit=10000
Content-Type: application/json
Authorization: Impersonate {{user}}

{
"languageItemCodes": ["DE"]
"text": "n1"
}

### Show assets search stats
Expand Down
4 changes: 3 additions & 1 deletion apps/server-asset-sg/src/features/assets/search/asset-search.service.spec.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,12 +32,14 @@ import { PrismaService } from '@/core/prisma.service';
import { fakeAssetPatch, fakeAssetUsage, fakeContact, fakeUser } from '@/features/asset-old/asset-edit.fake';
import { AssetData, AssetEditRepo } from '@/features/asset-old/asset-edit.repo';
import { AssetEditDetail } from '@/features/asset-old/asset-edit.service';
import { StudyRepo } from '@/features/studies/study.repo';

describe(AssetSearchService, () => {
const elastic = openElasticsearchClient();
const prisma = new PrismaService();
const assetRepo = new AssetEditRepo(prisma);
const service = new AssetSearchService(elastic, prisma, assetRepo);
const studyRepo = new StudyRepo(prisma);
const service = new AssetSearchService(elastic, prisma, assetRepo, studyRepo);

beforeAll(async () => {
const existsIndex = await elastic.indices.exists({ index: ASSET_ELASTIC_INDEX });
Expand Down
Loading

0 comments on commit 5bfbc69

Please sign in to comment.