refactor(service): implementations for querying observations

service/src/adapters/base/adapters.base.db.mongoose.ts

@@ -10,7 +10,7 @@ export type WithMongooseDefaultVersionKey = { [MongooseDefaultVersionKey]: numbe
 /**
  * Map Mongoose `Document` instances to plain entity objects.
  */
-export type DocumentMapping<DocType extends mongoose.AnyObject, E extends object> = (doc: DocType | mongoose.HydratedDocument<DocType>) => E
+export type DocumentMapping<DocType extends mongoose.AnyObject, E extends object> = (doc: DocType | mongoose.HydratedDocument<DocType> | mongoose.LeanDocument<DocType>) => E
 /**
  * Map entities to objects suitable to create Mongoose `Model` `Document` instances, as
  * in `new mongoose.Model(stub)`.

service/src/adapters/observations/adapters.observations.db.mongoose.ts

@@ -1,5 +1,5 @@
 import { MageEvent, MageEventAttrs, MageEventId } from '../../entities/events/entities.events'
-import { Attachment, AttachmentId, AttachmentNotFoundError, AttachmentPatchAttrs, copyObservationAttrs, EventScopedObservationRepository, FormEntry, FormEntryId, Observation, ObservationAttrs, ObservationFeatureProperties, ObservationId, ObservationImportantFlag, ObservationRepositoryError, ObservationRepositoryErrorCode, ObservationState, patchAttachment, Thumbnail } from '../../entities/observations/entities.observations'
+import { Attachment, AttachmentId, AttachmentNotFoundError, AttachmentPatchAttrs, copyObservationAttrs, EventScopedObservationRepository, UsersExpandedObservationAttrs, FindObservationsSpec, FormEntry, FormEntryId, Observation, ObservationAttrs, ObservationFeatureProperties, ObservationId, ObservationImportantFlag, ObservationRepositoryError, ObservationRepositoryErrorCode, ObservationState, patchAttachment, Thumbnail, UserExpandedObservationImportantFlag } from '../../entities/observations/entities.observations'
 import { BaseMongooseRepository, DocumentMapping, pageQuery, WithMongooseDefaultVersionKey } from '../base/adapters.base.db.mongoose'
 import mongoose from 'mongoose'
 import { MageEventDocument } from '../../models/event'
@@ -293,8 +293,7 @@ export class MongooseObservationRepository extends BaseMongooseRepository<Observ
         console.warn(`attempted to modify create timestamp on observation ${beforeDoc.id} from ${beforeDoc.createdAt} to ${docSeed.createdAt}`)
         docSeed.createdAt = new Date(beforeDoc.createdAt)
       }
-      //TODO remove any, was as legacy.ObservationDocument
-      beforeDoc = beforeDoc.set(docSeed) as any
+      beforeDoc = beforeDoc.set(docSeed)
     }
     else {
       const idVerified = await this.idModel.findById(dbId)
@@ -329,14 +328,70 @@ export class MongooseObservationRepository extends BaseMongooseRepository<Observ
     return latest ? this.entityForDocument(latest) : null
   }
 
+  async findSome<
+    FindSpec extends FindObservationsSpec = FindObservationsSpec,
+    T = FindSpec extends { populateUserNames: true } ? UsersExpandedObservationAttrs : ObservationAttrs
+  >(findSpec: FindSpec, mapping?: (x: ObservationAttrs | UsersExpandedObservationAttrs) => T): Promise<PageOf<T>> {
+    const { where, orderBy, paging: specPaging } = findSpec
+    const dbFilter = {} as any
+    if (where.lastModifiedAfter) {
+      dbFilter.lastModified = { $gte: where.lastModifiedAfter }
+    }
+    if (where.lastModifiedBefore) {
+      dbFilter.lastModified = { ...dbFilter.lastModified, $lt: where.lastModifiedBefore }
+    }
+    if (where.timestampAfter) {
+      dbFilter['properties.timestamp'] = { $gte: where.timestampAfter }
+    }
+    if (where.timestampBefore) {
+      dbFilter['properties.timestamp'] = { ...dbFilter['properties.timestamp'], $lt: where.timestampBefore }
+    }
+    if (where.stateIsAnyOf) {
+      dbFilter['states.0.name'] = { $in: where.stateIsAnyOf }
+    }
+    if (typeof where.isFlaggedImportant === 'boolean') {
+      dbFilter.important = { $exists: where.isFlaggedImportant }
+    }
+    if (Array.isArray(where.isFavoriteOfUsers) && where.isFavoriteOfUsers.length > 0) {
+      dbFilter.favoriteUserIds = { $in: where.isFavoriteOfUsers }
+    }
+    const dbSort = {} as any
+    const order = typeof orderBy?.order === 'number' ? orderBy.order : 1
+    if (orderBy?.field === 'lastModified') {
+      dbSort.lastModified = order
+    }
+    else if (orderBy?.field === 'timestamp') {
+      dbSort['properties.timestamp'] = order
+    }
+    // add _id to sort for consistent ordering
+    dbSort._id = orderBy?.order || -1
+    const populatedUserNullSafetyTransform = (user: object | null, _id: mongoose.Types.ObjectId): object => user ? user : { _id }
+    const options: mongoose.QueryOptions<ObservationDocument> = dbSort ? { sort: dbSort } : {}
+    const paging: PagingParameters = specPaging || { includeTotalCount: false, pageSize: Number.MAX_SAFE_INTEGER, pageIndex: 0 }
+    const counted = await pageQuery(
+      this.model.find(dbFilter, null, options)
+        .lean(true)
+        .populate([
+          // TODO: something smarter than a mongoose join would be good here
+          // maybe just store the display name on the observation document
+          { path: 'userId', select: 'displayName', transform: populatedUserNullSafetyTransform },
+          { path: 'important.userId', select: 'displayName', transform: populatedUserNullSafetyTransform }
+        ]),
+      paging
+    )
+    const mapResult = typeof mapping === 'function' ? mapping : ((x: any): T => x as T)
+    const items = await counted.query
+    const mappedItems = items.map(doc => mapResult(this.entityForDocument(doc)))
+    return pageOf(mappedItems, paging, counted.totalCount)
+  }
+
   async findLastModifiedAfter(timestamp: number, paging: PagingParameters): Promise<PageOf<ObservationAttrs>> {
     const match = { lastModified: {$gte: new Date(timestamp)} }
     const counted = await pageQuery(this.model.find(match), paging)
     const observations: ObservationAttrs[] = []
     for await (const doc of counted.query.cursor()) {
       observations.push(this.entityForDocument(doc))
     }
-
     return pageOf(observations, paging, counted.totalCount)
   }
 
@@ -365,36 +420,47 @@ export class MongooseObservationRepository extends BaseMongooseRepository<Observ
   }
 }
 
-export function docToEntity(doc: ObservationDocument, eventId: MageEventId): ObservationAttrs {
-  return createDocumentMapping(eventId)(doc)
+type PopulatedUserDocument = { _id: mongoose.Types.ObjectId, displayName: string }
+type ObservationDocumentPopulated = Omit<ObservationDocument, 'userId' | 'important'> & {
+  userId: PopulatedUserDocument
+  important: Omit<ObservationDocumentImportantFlag, 'userId'> & { userId: PopulatedUserDocument }
 }
 
-function createDocumentMapping(eventId: MageEventId): DocumentMapping<ObservationDocument, ObservationAttrs> {
+function createDocumentMapping(eventId: MageEventId): DocumentMapping<ObservationDocument | ObservationDocumentPopulated, ObservationAttrs> {
   return doc => {
-    const attrs: ObservationAttrs = {
+    const attrs: UsersExpandedObservationAttrs = {
       id: doc._id.toHexString(),
       eventId,
       createdAt: doc.createdAt,
       lastModified: doc.lastModified,
       type: doc.type,
       geometry: doc.geometry,
-      bbox: doc.bbox,
+      bbox: doc.bbox as GeoJSON.BBox,
       states: doc.states.map(stateAttrsForDoc),
       properties: {
         ...doc.properties,
         forms: doc.properties.forms.map(formEntryForDoc)
       },
+      important: importantFlagAttrsForDoc(doc) as any,
       attachments: doc.attachments.map(attachmentAttrsForDoc),
-      userId: doc.userId?.toHexString(),
       deviceId: doc.deviceId?.toHexString(),
-      important: importantFlagAttrsForDoc(doc),
       favoriteUserIds: doc.favoriteUserIds?.map(x => x.toHexString()),
     }
+    if (doc.userId instanceof mongoose.Types.ObjectId) {
+      attrs.userId = doc.userId.toHexString()
+    }
+    else if (doc.userId?._id) {
+      attrs.userId = doc.userId._id.toHexString()
+      attrs.user = {
+        id: attrs.userId,
+        displayName: doc.userId.displayName
+      }
+    }
     return attrs
   }
 }
 
-function importantFlagAttrsForDoc(doc: ObservationDocument): ObservationImportantFlag | undefined {
+function importantFlagAttrsForDoc(doc: ObservationDocument | ObservationDocumentPopulated | mongoose.LeanDocument<ObservationDocument | ObservationDocumentPopulated>): ObservationImportantFlag | UserExpandedObservationImportantFlag | undefined {
   /*
   because the observation schema defines `important` as a nested document
   instead of a subdocument schema, a mongoose observation document instance
@@ -405,11 +471,21 @@ function importantFlagAttrsForDoc(doc: ObservationDocument): ObservationImportan
   */
   const docImportant = doc.important
   if (docImportant?.userId || docImportant?.timestamp || docImportant?.description) {
-    return {
-      userId: docImportant.userId?.toHexString(),
+    const important: UserExpandedObservationImportantFlag = {
       timestamp: docImportant.timestamp,
       description: docImportant.description
     }
+    if (docImportant.userId instanceof mongoose.Types.ObjectId) {
+      important.userId = docImportant.userId.toHexString()
+    }
+    else if (docImportant.userId?._id instanceof mongoose.Types.ObjectId) {
+      important.userId = docImportant.userId._id.toHexString()
+      important.user = {
+        id: docImportant.userId._id.toHexString(),
+        displayName: docImportant.userId.displayName
+      }
+    }
+    return important
   }
   return void(0)
 }

service/src/app.api/observations/app.api.observations.ts

@@ -1,8 +1,9 @@
 import { EntityNotFoundError, InfrastructureError, InvalidInputError, PermissionDeniedError } from '../app.api.errors'
 import { AppRequest, AppRequestContext, AppResponse } from '../app.api.global'
-import { Attachment, AttachmentId, copyObservationAttrs, EventScopedObservationRepository, FormEntry, FormFieldEntry, Observation, ObservationAttrs, ObservationFeatureProperties, ObservationId, ObservationImportantFlag, ObservationState, StagedAttachmentContentRef, Thumbnail, thumbnailIndexForTargetDimension } from '../../entities/observations/entities.observations'
+import { Attachment, AttachmentId, copyObservationAttrs, EventScopedObservationRepository, FindObservationsSpec, FormEntry, FormFieldEntry, Observation, ObservationAttrs, ObservationFeatureProperties, ObservationId, ObservationImportantFlag, ObservationState, StagedAttachmentContentRef, Thumbnail, thumbnailIndexForTargetDimension } from '../../entities/observations/entities.observations'
 import { MageEvent } from '../../entities/events/entities.events'
 import { User, UserId } from '../../entities/users/entities.users'
+import { PageOf } from '../../entities/entities.global'
 
 
 
@@ -44,26 +45,11 @@ export interface ReadObservation {
 }
 
 export interface ReadObservationsRequest extends ObservationRequest {
-  filter: {
-    lastModifiedAfter?: Date | undefined,
-    lastModifiedBefore?: Date | undefined,
-    timestampAfter?: Date | undefined,
-    timestampBefore?: Date | undefined,
-    bbox?: GeoJSON.BBox,
-    states?: ObservationState['name'][] | undefined,
-  },
-  sort?: {
-    field: string,
-    order?: 1 | -1,
-  },
-  /**
-   * If `true`, populate the user names for the observation {@link ObservationAttrs.userId creator} and
-   * {@link ObservationImportantFlag.userId important flag}.
-   */
-  populate?: boolean | undefined
+  findSpec: FindObservationsSpec,
+  mapping?: <T>(x: ExoObservation) => T
 }
 export interface ReadObservations {
-  (req: ReadObservationsRequest): Promise<AppResponse<ExoObservation[], PermissionDeniedError | InvalidInputError>>
+  (req: ReadObservationsRequest): Promise<AppResponse<PageOf<ExoObservation[]>, PermissionDeniedError | InvalidInputError | InfrastructureError>>
 }
 
 export interface StoreAttachmentContent {

service/src/app.impl/observations/app.impl.observations.ts

@@ -52,6 +52,23 @@ export function SaveObservation(permissionService: api.ObservationPermissionServ
   }
 }
 
+export function ReadObservations(permissionService: api.ObservationPermissionService): api.ReadObservations {
+  return async function readObservations(req: api.ReadObservationsRequest): ReturnType<api.ReadObservations> {
+    const denied = await permissionService.ensureReadObservationPermission(req.context)
+    if (denied) {
+      return AppResponse.error(denied)
+    }
+    const mapping = (x: ObservationAttrs): any => (typeof req.mapping === 'function' ? req.mapping(api.exoObservationFor(x)) : api.exoObservationFor(x))
+    try {
+      const results = await req.context.observationRepository.findSome(req.findSpec, mapping)
+      return AppResponse.success(results)
+    }
+    catch (err) {
+      return AppResponse.error(infrastructureError(err instanceof Error ? err : String(err)))
+    }
+  }
+}
+
 export function ReadObservation(permissionService: api.ObservationPermissionService): api.ReadObservation {
   return async function readObservation(req: api.ReadObservationRequest): ReturnType<api.ReadObservation> {
     const denied = await permissionService.ensureReadObservationPermission(req.context)

service/src/entities/observations/entities.observations.ts

@@ -1,4 +1,4 @@
-import { UserId } from '../users/entities.users'
+import { User, UserId } from '../users/entities.users'
 import { BBox, Feature, Geometry } from 'geojson'
 import { MageEvent, MageEventAttrs, MageEventId } from '../events/entities.events'
 import { PageOf, PagingParameters, PendingEntityId } from '../entities.global'
@@ -630,10 +630,10 @@ export function validationResultMessage(result: ObservationValidationResult): st
   if (totalFormCountError) {
     errList.push(`${bulletPoint} ${totalFormCountError.message()}`)
   }
-  for (const [ formId, err ] of formCountErrors) {
+  for (const [ , err ] of formCountErrors) {
     errList.push(`${bulletPoint} ${err.message()}`)
   }
-  for (const [ formEntryId, formEntryErr ] of formEntryErrors) {
+  for (const [ , formEntryErr ] of formEntryErrors) {
     errList.push(`${bulletPoint} Form entry ${formEntryErr.formEntryPosition + 1} (${formEntryErr.formName}) is invalid.`)
     for (const fieldErr of formEntryErr.fieldErrors.values()) {
       errList.push(`  ${bulletPoint} ${fieldErr.message}`)
@@ -857,7 +857,13 @@ export interface EventScopedObservationRepository {
    * @returns an `Observation` object or `null`
    */
   findLatest(): Promise<ObservationAttrs | null>
+  /**
+   * @deprecated TODO: `findSome()` makes this obsolete.  One of the plugins might use this one.
+   */
   findLastModifiedAfter(timestamp: number, paging: PagingParameters): Promise<PageOf<ObservationAttrs>>
+  findSome<T = ObservationAttrs>(findSpec: FindObservationsSpecUnpopulated, mapping?: (o: ObservationAttrs) => T): Promise<PageOf<T>>
+  findSome<T = UsersExpandedObservationAttrs>(findSpec: FindObservationsSpecPopulated, mapping?: (o: UsersExpandedObservationAttrs) => T): Promise<PageOf<T>>
+  findSome<T = ObservationAttrs>(findSpec: FindObservationsSpec, mapping?: (o: ObservationAttrs | UsersExpandedObservationAttrs) => T): Promise<PageOf<T>>
   /**
    * Update the specified attachment with the given attributes.  This
    * persistence function exists alongside the {@link save} method to prevent
@@ -891,8 +897,59 @@ export interface EventScopedObservationRepository {
   nextAttachmentIds(count?: number): Promise<AttachmentId[]>
 }
 
-export class ObservationRepositoryError extends Error {
+export type FindObservationsSortField = 'lastModified' | 'timestamp'
 
+export interface FindObservationsSort {
+  /**
+   * The default sort field is `lastModified`.
+   */
+  field: FindObservationsSortField
+  /**
+   * `1` indicates ascending, `-1` indicates descending.  Ascending is the default order.
+   */
+  order?: 1 | -1
+}
+export interface FindObservationsSpec {
+  where: {
+    lastModifiedAfter?: Date
+    lastModifiedBefore?: Date
+    timestampAfter?: Date
+    timestampBefore?: Date
+    /**
+     * A series of lon/lat coordinates in the order [ west, south, east, north ] as in
+     * https://datatracker.ietf.org/doc/html/rfc7946#section-5.
+     */
+    locationIntersects?: [ number, number, number, number ]
+    stateIsAnyOf?: ObservationStateName[]
+    /**
+     * Specifying a boolean value finds only observations conforming to the value, whereas omitting or specifiying
+     * `null` causes the query to disregard the important flag.
+     */
+    isFlaggedImportant?: boolean | null,
+    isFavoriteOfUsers?: UserId[],
+  }
+  /**
+   * If `true`, populate the display names from related user documents on the observation creator and important flag.
+   * This results in adding `user: { id: string, displayName: string }` entries on the observation document and
+   * important sub-document.
+   */
+  populateUserNames?: boolean
+  orderBy?: FindObservationsSort
+  paging?: PagingParameters
+}
+type FindObservationsSpecPopulated = FindObservationsSpec & { populateUserNames: true }
+type FindObservationsSpecUnpopulated = Omit<FindObservationsSpec, 'populateUserNames'> | (FindObservationsSpec & { populateUserNames: false })
+
+export type ObservationUserExpanded = Pick<User, 'id' | 'displayName'>
+
+export type UsersExpandedObservationAttrs = ObservationAttrs & {
+  user?: ObservationUserExpanded
+  important?: UserExpandedObservationImportantFlag
+}
+
+export type UserExpandedObservationImportantFlag = ObservationImportantFlag & { user?: ObservationUserExpanded }
+
+export class ObservationRepositoryError extends Error {
   constructor(readonly code: ObservationRepositoryErrorCode, message?: string) {
     super(message)
   }
@@ -1186,7 +1243,7 @@ const FieldTypeValidationRules: { [type in FormFieldType]: FormFieldValidationRu
   [FormFieldType.Email]: validateRequiredThen(context => fields.email.EmailFieldValidation(context.field, context.fieldEntry, FormFieldValidationResult(context))),
   [FormFieldType.Geometry]: validateRequiredThen(context => fields.geometry.GeometryFieldValidation(context.field, context.fieldEntry, FormFieldValidationResult(context))),
   // TODO: no validation at all? legacy validation code did nothing for hidden fields
-  [FormFieldType.Hidden]: context => null,
+  [FormFieldType.Hidden]: () => null,
   [FormFieldType.MultiSelectDropdown]: validateRequiredThen(context => fields.multiselect.MultiSelectFormFieldValidation(context.field, context.fieldEntry, FormFieldValidationResult(context))),
   [FormFieldType.Numeric]: validateRequiredThen(context => fields.numeric.NumericFieldValidation(context.field, context.fieldEntry, FormFieldValidationResult(context))),
   [FormFieldType.Password]: validateRequiredThen(context => fields.text.TextFieldValidation(context.field, context.fieldEntry, FormFieldValidationResult(context))),