From 955a5c14d3db4a37f7c1dc8f522beaa10eda4a6e Mon Sep 17 00:00:00 2001 From: "Robert St. John" Date: Mon, 12 Aug 2024 13:55:31 -0600 Subject: [PATCH] refactor(service): implementations for querying observations --- .../base/adapters.base.db.mongoose.ts | 2 +- .../adapters.observations.db.mongoose.ts | 104 +++++++++++++++--- .../observations/app.api.observations.ts | 24 +--- .../observations/app.impl.observations.ts | 17 +++ .../observations/entities.observations.ts | 67 ++++++++++- .../adapters.observations.db.mongoose.test.ts | 29 +++-- .../app/observations/app.observations.test.ts | 7 ++ 7 files changed, 203 insertions(+), 47 deletions(-) diff --git a/service/src/adapters/base/adapters.base.db.mongoose.ts b/service/src/adapters/base/adapters.base.db.mongoose.ts index c4a3a4424..3401966e4 100644 --- a/service/src/adapters/base/adapters.base.db.mongoose.ts +++ b/service/src/adapters/base/adapters.base.db.mongoose.ts @@ -10,7 +10,7 @@ export type WithMongooseDefaultVersionKey = { [MongooseDefaultVersionKey]: numbe /** * Map Mongoose `Document` instances to plain entity objects. */ -export type DocumentMapping = (doc: DocType | mongoose.HydratedDocument) => E +export type DocumentMapping = (doc: DocType | mongoose.HydratedDocument | mongoose.LeanDocument) => E /** * Map entities to objects suitable to create Mongoose `Model` `Document` instances, as * in `new mongoose.Model(stub)`. diff --git a/service/src/adapters/observations/adapters.observations.db.mongoose.ts b/service/src/adapters/observations/adapters.observations.db.mongoose.ts index 5a6324a9e..de9d48274 100644 --- a/service/src/adapters/observations/adapters.observations.db.mongoose.ts +++ b/service/src/adapters/observations/adapters.observations.db.mongoose.ts @@ -1,5 +1,5 @@ import { MageEvent, MageEventAttrs, MageEventId } from '../../entities/events/entities.events' -import { Attachment, AttachmentId, AttachmentNotFoundError, AttachmentPatchAttrs, copyObservationAttrs, EventScopedObservationRepository, FormEntry, FormEntryId, Observation, ObservationAttrs, ObservationFeatureProperties, ObservationId, ObservationImportantFlag, ObservationRepositoryError, ObservationRepositoryErrorCode, ObservationState, patchAttachment, Thumbnail } from '../../entities/observations/entities.observations' +import { Attachment, AttachmentId, AttachmentNotFoundError, AttachmentPatchAttrs, copyObservationAttrs, EventScopedObservationRepository, UsersExpandedObservationAttrs, FindObservationsSpec, FormEntry, FormEntryId, Observation, ObservationAttrs, ObservationFeatureProperties, ObservationId, ObservationImportantFlag, ObservationRepositoryError, ObservationRepositoryErrorCode, ObservationState, patchAttachment, Thumbnail, UserExpandedObservationImportantFlag } from '../../entities/observations/entities.observations' import { BaseMongooseRepository, DocumentMapping, pageQuery, WithMongooseDefaultVersionKey } from '../base/adapters.base.db.mongoose' import mongoose from 'mongoose' import { MageEventDocument } from '../../models/event' @@ -293,8 +293,7 @@ export class MongooseObservationRepository extends BaseMongooseRepository(findSpec: FindSpec, mapping?: (x: ObservationAttrs | UsersExpandedObservationAttrs) => T): Promise> { + const { where, orderBy, paging: specPaging } = findSpec + const dbFilter = {} as any + if (where.lastModifiedAfter) { + dbFilter.lastModified = { $gte: where.lastModifiedAfter } + } + if (where.lastModifiedBefore) { + dbFilter.lastModified = { ...dbFilter.lastModified, $lt: where.lastModifiedBefore } + } + if (where.timestampAfter) { + dbFilter['properties.timestamp'] = { $gte: where.timestampAfter } + } + if (where.timestampBefore) { + dbFilter['properties.timestamp'] = { ...dbFilter['properties.timestamp'], $lt: where.timestampBefore } + } + if (where.stateIsAnyOf) { + dbFilter['states.0.name'] = { $in: where.stateIsAnyOf } + } + if (typeof where.isFlaggedImportant === 'boolean') { + dbFilter.important = { $exists: where.isFlaggedImportant } + } + if (Array.isArray(where.isFavoriteOfUsers) && where.isFavoriteOfUsers.length > 0) { + dbFilter.favoriteUserIds = { $in: where.isFavoriteOfUsers } + } + const dbSort = {} as any + const order = typeof orderBy?.order === 'number' ? orderBy.order : 1 + if (orderBy?.field === 'lastModified') { + dbSort.lastModified = order + } + else if (orderBy?.field === 'timestamp') { + dbSort['properties.timestamp'] = order + } + // add _id to sort for consistent ordering + dbSort._id = orderBy?.order || -1 + const populatedUserNullSafetyTransform = (user: object | null, _id: mongoose.Types.ObjectId): object => user ? user : { _id } + const options: mongoose.QueryOptions = dbSort ? { sort: dbSort } : {} + const paging: PagingParameters = specPaging || { includeTotalCount: false, pageSize: Number.MAX_SAFE_INTEGER, pageIndex: 0 } + const counted = await pageQuery( + this.model.find(dbFilter, null, options) + .lean(true) + .populate([ + // TODO: something smarter than a mongoose join would be good here + // maybe just store the display name on the observation document + { path: 'userId', select: 'displayName', transform: populatedUserNullSafetyTransform }, + { path: 'important.userId', select: 'displayName', transform: populatedUserNullSafetyTransform } + ]), + paging + ) + const mapResult = typeof mapping === 'function' ? mapping : ((x: any): T => x as T) + const items = await counted.query + const mappedItems = items.map(doc => mapResult(this.entityForDocument(doc))) + return pageOf(mappedItems, paging, counted.totalCount) + } + async findLastModifiedAfter(timestamp: number, paging: PagingParameters): Promise> { const match = { lastModified: {$gte: new Date(timestamp)} } const counted = await pageQuery(this.model.find(match), paging) @@ -336,7 +392,6 @@ export class MongooseObservationRepository extends BaseMongooseRepository & { + userId: PopulatedUserDocument + important: Omit & { userId: PopulatedUserDocument } } -function createDocumentMapping(eventId: MageEventId): DocumentMapping { +function createDocumentMapping(eventId: MageEventId): DocumentMapping { return doc => { - const attrs: ObservationAttrs = { + const attrs: UsersExpandedObservationAttrs = { id: doc._id.toHexString(), eventId, createdAt: doc.createdAt, lastModified: doc.lastModified, type: doc.type, geometry: doc.geometry, - bbox: doc.bbox, + bbox: doc.bbox as GeoJSON.BBox, states: doc.states.map(stateAttrsForDoc), properties: { ...doc.properties, forms: doc.properties.forms.map(formEntryForDoc) }, + important: importantFlagAttrsForDoc(doc) as any, attachments: doc.attachments.map(attachmentAttrsForDoc), - userId: doc.userId?.toHexString(), deviceId: doc.deviceId?.toHexString(), - important: importantFlagAttrsForDoc(doc), favoriteUserIds: doc.favoriteUserIds?.map(x => x.toHexString()), } + if (doc.userId instanceof mongoose.Types.ObjectId) { + attrs.userId = doc.userId.toHexString() + } + else if (doc.userId?._id) { + attrs.userId = doc.userId._id.toHexString() + attrs.user = { + id: attrs.userId, + displayName: doc.userId.displayName + } + } return attrs } } -function importantFlagAttrsForDoc(doc: ObservationDocument): ObservationImportantFlag | undefined { +function importantFlagAttrsForDoc(doc: ObservationDocument | ObservationDocumentPopulated | mongoose.LeanDocument): ObservationImportantFlag | UserExpandedObservationImportantFlag | undefined { /* because the observation schema defines `important` as a nested document instead of a subdocument schema, a mongoose observation document instance @@ -405,11 +471,21 @@ function importantFlagAttrsForDoc(doc: ObservationDocument): ObservationImportan */ const docImportant = doc.important if (docImportant?.userId || docImportant?.timestamp || docImportant?.description) { - return { - userId: docImportant.userId?.toHexString(), + const important: UserExpandedObservationImportantFlag = { timestamp: docImportant.timestamp, description: docImportant.description } + if (docImportant.userId instanceof mongoose.Types.ObjectId) { + important.userId = docImportant.userId.toHexString() + } + else if (docImportant.userId?._id instanceof mongoose.Types.ObjectId) { + important.userId = docImportant.userId._id.toHexString() + important.user = { + id: docImportant.userId._id.toHexString(), + displayName: docImportant.userId.displayName + } + } + return important } return void(0) } diff --git a/service/src/app.api/observations/app.api.observations.ts b/service/src/app.api/observations/app.api.observations.ts index bf6871627..cd61c6cdb 100644 --- a/service/src/app.api/observations/app.api.observations.ts +++ b/service/src/app.api/observations/app.api.observations.ts @@ -1,8 +1,9 @@ import { EntityNotFoundError, InfrastructureError, InvalidInputError, PermissionDeniedError } from '../app.api.errors' import { AppRequest, AppRequestContext, AppResponse } from '../app.api.global' -import { Attachment, AttachmentId, copyObservationAttrs, EventScopedObservationRepository, FormEntry, FormFieldEntry, Observation, ObservationAttrs, ObservationFeatureProperties, ObservationId, ObservationImportantFlag, ObservationState, StagedAttachmentContentRef, Thumbnail, thumbnailIndexForTargetDimension } from '../../entities/observations/entities.observations' +import { Attachment, AttachmentId, copyObservationAttrs, EventScopedObservationRepository, FindObservationsSpec, FormEntry, FormFieldEntry, Observation, ObservationAttrs, ObservationFeatureProperties, ObservationId, ObservationImportantFlag, ObservationState, StagedAttachmentContentRef, Thumbnail, thumbnailIndexForTargetDimension } from '../../entities/observations/entities.observations' import { MageEvent } from '../../entities/events/entities.events' import { User, UserId } from '../../entities/users/entities.users' +import { PageOf } from '../../entities/entities.global' @@ -44,26 +45,11 @@ export interface ReadObservation { } export interface ReadObservationsRequest extends ObservationRequest { - filter: { - lastModifiedAfter?: Date | undefined, - lastModifiedBefore?: Date | undefined, - timestampAfter?: Date | undefined, - timestampBefore?: Date | undefined, - bbox?: GeoJSON.BBox, - states?: ObservationState['name'][] | undefined, - }, - sort?: { - field: string, - order?: 1 | -1, - }, - /** - * If `true`, populate the user names for the observation {@link ObservationAttrs.userId creator} and - * {@link ObservationImportantFlag.userId important flag}. - */ - populate?: boolean | undefined + findSpec: FindObservationsSpec, + mapping?: (x: ExoObservation) => T } export interface ReadObservations { - (req: ReadObservationsRequest): Promise> + (req: ReadObservationsRequest): Promise, PermissionDeniedError | InvalidInputError | InfrastructureError>> } export interface StoreAttachmentContent { diff --git a/service/src/app.impl/observations/app.impl.observations.ts b/service/src/app.impl/observations/app.impl.observations.ts index 6135f3dfb..1f44eeb3f 100644 --- a/service/src/app.impl/observations/app.impl.observations.ts +++ b/service/src/app.impl/observations/app.impl.observations.ts @@ -52,6 +52,23 @@ export function SaveObservation(permissionService: api.ObservationPermissionServ } } +export function ReadObservations(permissionService: api.ObservationPermissionService): api.ReadObservations { + return async function readObservations(req: api.ReadObservationsRequest): ReturnType { + const denied = await permissionService.ensureReadObservationPermission(req.context) + if (denied) { + return AppResponse.error(denied) + } + const mapping = (x: ObservationAttrs): any => (typeof req.mapping === 'function' ? req.mapping(api.exoObservationFor(x)) : api.exoObservationFor(x)) + try { + const results = await req.context.observationRepository.findSome(req.findSpec, mapping) + return AppResponse.success(results) + } + catch (err) { + return AppResponse.error(infrastructureError(err instanceof Error ? err : String(err))) + } + } +} + export function ReadObservation(permissionService: api.ObservationPermissionService): api.ReadObservation { return async function readObservation(req: api.ReadObservationRequest): ReturnType { const denied = await permissionService.ensureReadObservationPermission(req.context) diff --git a/service/src/entities/observations/entities.observations.ts b/service/src/entities/observations/entities.observations.ts index 00d5c5f86..1091cfc59 100644 --- a/service/src/entities/observations/entities.observations.ts +++ b/service/src/entities/observations/entities.observations.ts @@ -1,4 +1,4 @@ -import { UserId } from '../users/entities.users' +import { User, UserId } from '../users/entities.users' import { BBox, Feature, Geometry } from 'geojson' import { MageEvent, MageEventAttrs, MageEventId } from '../events/entities.events' import { PageOf, PagingParameters, PendingEntityId } from '../entities.global' @@ -630,10 +630,10 @@ export function validationResultMessage(result: ObservationValidationResult): st if (totalFormCountError) { errList.push(`${bulletPoint} ${totalFormCountError.message()}`) } - for (const [ formId, err ] of formCountErrors) { + for (const [ , err ] of formCountErrors) { errList.push(`${bulletPoint} ${err.message()}`) } - for (const [ formEntryId, formEntryErr ] of formEntryErrors) { + for (const [ , formEntryErr ] of formEntryErrors) { errList.push(`${bulletPoint} Form entry ${formEntryErr.formEntryPosition + 1} (${formEntryErr.formName}) is invalid.`) for (const fieldErr of formEntryErr.fieldErrors.values()) { errList.push(` ${bulletPoint} ${fieldErr.message}`) @@ -857,7 +857,13 @@ export interface EventScopedObservationRepository { * @returns an `Observation` object or `null` */ findLatest(): Promise + /** + * @deprecated TODO: `findSome()` makes this obsolete. One of the plugins might use this one. + */ findLastModifiedAfter(timestamp: number, paging: PagingParameters): Promise> + findSome(findSpec: FindObservationsSpecUnpopulated, mapping?: (o: ObservationAttrs) => T): Promise> + findSome(findSpec: FindObservationsSpecPopulated, mapping?: (o: UsersExpandedObservationAttrs) => T): Promise> + findSome(findSpec: FindObservationsSpec, mapping?: (o: ObservationAttrs | UsersExpandedObservationAttrs) => T): Promise> /** * Update the specified attachment with the given attributes. This * persistence function exists alongside the {@link save} method to prevent @@ -891,8 +897,59 @@ export interface EventScopedObservationRepository { nextAttachmentIds(count?: number): Promise } -export class ObservationRepositoryError extends Error { +export type FindObservationsSortField = 'lastModified' | 'timestamp' +export interface FindObservationsSort { + /** + * The default sort field is `lastModified`. + */ + field: FindObservationsSortField + /** + * `1` indicates ascending, `-1` indicates descending. Ascending is the default order. + */ + order?: 1 | -1 +} +export interface FindObservationsSpec { + where: { + lastModifiedAfter?: Date + lastModifiedBefore?: Date + timestampAfter?: Date + timestampBefore?: Date + /** + * A series of lon/lat coordinates in the order [ west, south, east, north ] as in + * https://datatracker.ietf.org/doc/html/rfc7946#section-5. + */ + locationIntersects?: [ number, number, number, number ] + stateIsAnyOf?: ObservationStateName[] + /** + * Specifying a boolean value finds only observations conforming to the value, whereas omitting or specifiying + * `null` causes the query to disregard the important flag. + */ + isFlaggedImportant?: boolean | null, + isFavoriteOfUsers?: UserId[], + } + /** + * If `true`, populate the display names from related user documents on the observation creator and important flag. + * This results in adding `user: { id: string, displayName: string }` entries on the observation document and + * important sub-document. + */ + populateUserNames?: boolean + orderBy?: FindObservationsSort + paging?: PagingParameters +} +type FindObservationsSpecPopulated = FindObservationsSpec & { populateUserNames: true } +type FindObservationsSpecUnpopulated = Omit | (FindObservationsSpec & { populateUserNames: false }) + +export type ObservationUserExpanded = Pick + +export type UsersExpandedObservationAttrs = ObservationAttrs & { + user?: ObservationUserExpanded + important?: UserExpandedObservationImportantFlag +} + +export type UserExpandedObservationImportantFlag = ObservationImportantFlag & { user?: ObservationUserExpanded } + +export class ObservationRepositoryError extends Error { constructor(readonly code: ObservationRepositoryErrorCode, message?: string) { super(message) } @@ -1186,7 +1243,7 @@ const FieldTypeValidationRules: { [type in FormFieldType]: FormFieldValidationRu [FormFieldType.Email]: validateRequiredThen(context => fields.email.EmailFieldValidation(context.field, context.fieldEntry, FormFieldValidationResult(context))), [FormFieldType.Geometry]: validateRequiredThen(context => fields.geometry.GeometryFieldValidation(context.field, context.fieldEntry, FormFieldValidationResult(context))), // TODO: no validation at all? legacy validation code did nothing for hidden fields - [FormFieldType.Hidden]: context => null, + [FormFieldType.Hidden]: () => null, [FormFieldType.MultiSelectDropdown]: validateRequiredThen(context => fields.multiselect.MultiSelectFormFieldValidation(context.field, context.fieldEntry, FormFieldValidationResult(context))), [FormFieldType.Numeric]: validateRequiredThen(context => fields.numeric.NumericFieldValidation(context.field, context.fieldEntry, FormFieldValidationResult(context))), [FormFieldType.Password]: validateRequiredThen(context => fields.text.TextFieldValidation(context.field, context.fieldEntry, FormFieldValidationResult(context))), diff --git a/service/test/adapters/observations/adapters.observations.db.mongoose.test.ts b/service/test/adapters/observations/adapters.observations.db.mongoose.test.ts index 091ccf479..bf57d8e1b 100644 --- a/service/test/adapters/observations/adapters.observations.db.mongoose.test.ts +++ b/service/test/adapters/observations/adapters.observations.db.mongoose.test.ts @@ -3,13 +3,13 @@ import { expect } from 'chai' import mongoose from 'mongoose' import _ from 'lodash' import { MongooseMageEventRepository } from '../../../lib/adapters/events/adapters.events.db.mongoose' -import { MongooseObservationRepository, ObservationModel } from '../../../lib/adapters/observations/adapters.observations.db.mongoose' +import { MongooseObservationRepository, ObservationDocument, ObservationModel } from '../../../lib/adapters/observations/adapters.observations.db.mongoose' import * as legacy from '../../../lib/models/observation' import * as legacyEvent from '../../../lib/models/event' import { MageEventDocument } from '../../../src/models/event' import { MageEvent, MageEventAttrs, MageEventCreateAttrs, MageEventId } from '../../../lib/entities/events/entities.events' -import { ObservationAttrs, ObservationId, Observation, ObservationRepositoryError, ObservationRepositoryErrorCode, copyObservationAttrs, AttachmentContentPatchAttrs, copyAttachmentAttrs, AttachmentNotFoundError, AttachmentPatchAttrs, removeAttachment, validationResultMessage, ObservationDomainEventType, ObservationEmitted, PendingObservationDomainEvent, AttachmentsRemovedDomainEvent } from '../../../lib/entities/observations/entities.observations' +import { ObservationAttrs, ObservationId, Observation, ObservationRepositoryError, ObservationRepositoryErrorCode, copyObservationAttrs, AttachmentContentPatchAttrs, copyAttachmentAttrs, AttachmentNotFoundError, AttachmentPatchAttrs, removeAttachment, validationResultMessage, ObservationDomainEventType, ObservationEmitted, PendingObservationDomainEvent, AttachmentsRemovedDomainEvent, ObservationStateName, EventScopedObservationRepository } from '../../../lib/entities/observations/entities.observations' import { AttachmentPresentationType, FormFieldType, Form, AttachmentMediaTypes } from '../../../lib/entities/events/entities.events.forms' import util from 'util' import { PendingEntityId } from '../../../lib/entities/entities.global' @@ -157,6 +157,19 @@ describe('mongoose observation repository', function() { }) }) + describe('reading observations', function() { + + describe('finding some', function() { + + it('has tests', async function() { + + const some = await repo.findSome({ where: {} }) + + expect.fail('todo') + }) + }) + }) + describe('saving observations', function() { describe('new observations', function() { @@ -241,12 +254,12 @@ describe('mongoose observation repository', function() { attrs.states = [ { id: (new mongoose.Types.ObjectId()).toHexString(), - name: 'active', + name: ObservationStateName.Active, userId: (new mongoose.Types.ObjectId()).toHexString() }, { id: (new mongoose.Types.ObjectId()).toHexString(), - name: 'archive', + name: ObservationStateName.Archived, userId: undefined } ] @@ -292,7 +305,7 @@ describe('mongoose observation repository', function() { } ] origAttrs.states = [ - { id: (new mongoose.Types.ObjectId()).toHexString(), name: 'active', userId: (new mongoose.Types.ObjectId()).toHexString() } + { id: (new mongoose.Types.ObjectId()).toHexString(), name: ObservationStateName.Active, userId: (new mongoose.Types.ObjectId()).toHexString() } ] origAttrs.properties.forms = [ { @@ -325,7 +338,7 @@ describe('mongoose observation repository', function() { coordinates: [ 12, 34 ] } putAttrs.states = [ - { name: 'archive', id: PendingEntityId } + { name: ObservationStateName.Archived, id: PendingEntityId } ] putAttrs.properties.forms = [ { @@ -411,7 +424,7 @@ describe('mongoose observation repository', function() { state1Stub.states = [ { id: PendingEntityId, - name: 'archive', + name: ObservationStateName.Archived, userId: (new mongoose.Types.ObjectId()).toHexString() } ] @@ -422,7 +435,7 @@ describe('mongoose observation repository', function() { state2Stub.states = [ { id: PendingEntityId, - name: 'active', + name: ObservationStateName.Active, userId: (new mongoose.Types.ObjectId()).toHexString() }, state1Saved.states[0], diff --git a/service/test/app/observations/app.observations.test.ts b/service/test/app/observations/app.observations.test.ts index 142fb381b..2da37bd29 100644 --- a/service/test/app/observations/app.observations.test.ts +++ b/service/test/app/observations/app.observations.test.ts @@ -1553,6 +1553,13 @@ describe('observations use case interactions', function() { describe('reading many', function() { it('has tests', async function() { + + const req: api.ReadObservationsRequest = { + context, + findSpec: { + where: {} + } + } expect.fail('todo') }) })