SIMSBIOHUB-619: Import Observations With Sampling Information

api/src/constants/dates.ts

@@ -0,0 +1,22 @@
+/*
+ * Date formats.
+ *
+ * See BC Gov standards: https://www2.gov.bc.ca/gov/content/governments/services-for-government/policies-procedures/web-content-development-guides/writing-for-the-web/web-style-guide/numbers
+ */
+export const DefaultDateFormat = 'YYYY-MM-DD'; // 2020-01-05
+
+export const DefaultDateFormatReverse = 'DD-MM-YYYY'; // 05-01-2020
+
+export const AltDateFormat = 'YYYY/MM/DD'; // 2020/01/05
+
+export const AltDateFormatReverse = 'DD/MM/YYYY'; // 05/01/2020
+
+/*
+ * Time formats.
+ */
+export const DefaultTimeFormat = 'HH:mm:ss'; // 23:00:00
+
+/*
+ * Datetime formats.
+ */
+export const DefaultDateTimeFormat = `${DefaultDateFormat}T${DefaultTimeFormat}`; // 2020-01-05T23:00:00

api/src/models/project-survey-attachments.ts

@@ -1,4 +1,4 @@
-import { default as dayjs } from 'dayjs';
+import dayjs from 'dayjs';
 import { ATTACHMENT_TYPE } from '../constants/attachments';
 import { getLogger } from '../utils/logger';
 import { SurveySupplementaryData } from './survey-view';

api/src/openapi/schemas/observation.ts

@@ -60,19 +60,27 @@ export const observervationsWithSubcountDataSchema: OpenAPIV3.SchemaObject = {
             nullable: true
           },
           latitude: {
-            type: 'number'
+            type: 'number',
+            nullable: true,
+            minimum: -90,
+            maximum: 90
           },
           longitude: {
-            type: 'number'
+            type: 'number',
+            nullable: true,
+            minimum: -180,
+            maximum: 180
           },
           count: {
             type: 'integer'
           },
           observation_date: {
-            type: 'string'
+            type: 'string',
+            nullable: true
           },
           observation_time: {
-            type: 'string'
+            type: 'string',
+            nullable: true
           },
           survey_sample_site_name: {
             type: 'string',

api/src/paths/project/{projectId}/survey/{surveyId}/deployments/index.ts

@@ -1,6 +1,7 @@
 import dayjs from 'dayjs';
 import { RequestHandler } from 'express';
 import { Operation } from 'express-openapi';
+import { DefaultDateFormat, DefaultTimeFormat } from '../../../../../../constants/dates';
 import { PROJECT_PERMISSION, SYSTEM_ROLE } from '../../../../../../constants/roles';
 import { getDBConnection } from '../../../../../../database/db';
 import { getDeploymentSchema } from '../../../../../../openapi/schemas/deployment';
@@ -204,16 +205,16 @@ export function getDeploymentsInSurvey(): RequestHandler {
           assignment_id: matchingBctwDeployments[0].assignment_id,
           collar_id: matchingBctwDeployments[0].collar_id,
           attachment_start_date: matchingBctwDeployments[0].attachment_start
-            ? dayjs(matchingBctwDeployments[0].attachment_start).format('YYYY-MM-DD')
+            ? dayjs(matchingBctwDeployments[0].attachment_start).format(DefaultDateFormat)
             : null,
           attachment_start_time: matchingBctwDeployments[0].attachment_start
-            ? dayjs(matchingBctwDeployments[0].attachment_start).format('HH:mm:ss')
+            ? dayjs(matchingBctwDeployments[0].attachment_start).format(DefaultTimeFormat)
             : null,
           attachment_end_date: matchingBctwDeployments[0].attachment_end
-            ? dayjs(matchingBctwDeployments[0].attachment_end).format('YYYY-MM-DD')
+            ? dayjs(matchingBctwDeployments[0].attachment_end).format(DefaultDateFormat)
             : null,
           attachment_end_time: matchingBctwDeployments[0].attachment_end
-            ? dayjs(matchingBctwDeployments[0].attachment_end).format('HH:mm:ss')
+            ? dayjs(matchingBctwDeployments[0].attachment_end).format(DefaultTimeFormat)
             : null,
           bctw_deployment_id: matchingBctwDeployments[0].deployment_id,
           device_id: matchingBctwDeployments[0].device_id,

api/src/paths/project/{projectId}/survey/{surveyId}/deployments/{deploymentId}/index.ts

@@ -2,6 +2,7 @@ import { AxiosError } from 'axios';
 import dayjs from 'dayjs';
 import { RequestHandler } from 'express';
 import { Operation } from 'express-openapi';
+import { DefaultDateFormat, DefaultTimeFormat } from '../../../../../../../constants/dates';
 import { PROJECT_PERMISSION, SYSTEM_ROLE } from '../../../../../../../constants/roles';
 import { getDBConnection } from '../../../../../../../database/db';
 import { HTTP400 } from '../../../../../../../errors/http-error';
@@ -211,16 +212,16 @@ export function getDeploymentById(): RequestHandler {
         assignment_id: matchingBctwDeployments[0].assignment_id,
         collar_id: matchingBctwDeployments[0].collar_id,
         attachment_start_date: matchingBctwDeployments[0].attachment_start
-          ? dayjs(matchingBctwDeployments[0].attachment_start).format('YYYY-MM-DD')
+          ? dayjs(matchingBctwDeployments[0].attachment_start).format(DefaultDateFormat)
           : null,
         attachment_start_time: matchingBctwDeployments[0].attachment_start
-          ? dayjs(matchingBctwDeployments[0].attachment_start).format('HH:mm:ss')
+          ? dayjs(matchingBctwDeployments[0].attachment_start).format(DefaultTimeFormat)
           : null,
         attachment_end_date: matchingBctwDeployments[0].attachment_end
-          ? dayjs(matchingBctwDeployments[0].attachment_end).format('YYYY-MM-DD')
+          ? dayjs(matchingBctwDeployments[0].attachment_end).format(DefaultDateFormat)
           : null,
         attachment_end_time: matchingBctwDeployments[0].attachment_end
-          ? dayjs(matchingBctwDeployments[0].attachment_end).format('HH:mm:ss')
+          ? dayjs(matchingBctwDeployments[0].attachment_end).format(DefaultTimeFormat)
           : null,
         bctw_deployment_id: matchingBctwDeployments[0].deployment_id,
         device_id: matchingBctwDeployments[0].device_id,

api/src/paths/project/{projectId}/survey/{surveyId}/observations/process.ts

@@ -68,7 +68,7 @@ POST.apiDoc = {
                 surveySamplePeriodId: {
                   type: 'integer',
                   description:
-                    'The optional ID of a survey sample period to associate the parsed observation records with.'
+                    'The optional ID of a survey sample period to associate the parsed observation records with. This is used when uploading all observations to a specific sample period, not when each record is for a different sample period.'
                 }
               }
             }

api/src/repositories/observation-repository/observation-repository.ts

@@ -31,11 +31,11 @@ export const ObservationRecord = z.object({
   survey_sample_site_id: z.number().nullable(),
   survey_sample_method_id: z.number().nullable(),
   survey_sample_period_id: z.number().nullable(),
-  latitude: z.number(),
-  longitude: z.number(),
+  latitude: z.number().nullable(),
+  longitude: z.number().nullable(),
   count: z.number(),
-  observation_time: z.string(),
-  observation_date: z.string(),
+  observation_time: z.string().nullable(),
+  observation_date: z.string().nullable(),
   create_date: z.string(),
   create_user: z.number(),
   update_date: z.string().nullable(),
@@ -320,10 +320,10 @@ export class ObservationRepository extends BaseRepository {
             observation.survey_sample_method_id ?? 'NULL',
             observation.survey_sample_period_id ?? 'NULL',
             observation.count,
-            observation.latitude,
-            observation.longitude,
-            `'${observation.observation_date}'`,
-            `'${observation.observation_time}'`,
+            observation.latitude ?? 'NULL',
+            observation.longitude ?? 'NULL',
+            observation.observation_date ? `'${observation.observation_date}'` : 'NULL',
+            observation.observation_time ? `'${observation.observation_time}'` : 'NULL',
             observation.itis_tsn ?? 'NULL',
             observation.itis_scientific_name ? `'${observation.itis_scientific_name}'` : 'NULL'
           ].join(', ')})`;
@@ -373,6 +373,9 @@ export class ObservationRepository extends BaseRepository {
         knex.raw("JSON_BUILD_OBJECT('type', 'Point', 'coordinates', JSON_BUILD_ARRAY(longitude, latitude)) as geometry")
       )
       .from('survey_observation')
+      // TODO: For observations without lat/lon, get a location from the sampling site?
+      .whereNotNull('latitude')
+      .whereNotNull('longitude')
       .where('survey_id', surveyId);
 
     const response = await this.connection.knex(query, ObservationGeometryRecord);

api/src/services/import-services/capture/import-captures-strategy.test.ts

@@ -22,18 +22,18 @@ describe('import-captures-service', () => {
         I1: { t: 's', v: 'RELEASE_LONGITUDE' },
         J1: { t: 's', v: 'RELEASE_COMMENT' },
         K1: { t: 's', v: 'CAPTURE_COMMENT' },
-        A2: { z: 'm/d/yy', t: 'd', v: '2024-10-10T07:00:00.000Z', w: '10/10/24' },
+        A2: { t: 's', v: '2024-10-11' },
         B2: { t: 's', v: 'Carl' },
         C2: { t: 's', v: '10:10:10' },
         D2: { t: 'n', w: '90', v: 90 },
         E2: { t: 'n', w: '100', v: 100 },
-        F2: { z: 'm/d/yy', t: 'd', v: '2024-10-10T07:00:00.000Z', w: '10/10/24' },
+        F2: { t: 's', v: '2024-10-10' },
         G2: { t: 's', v: '9:09' },
         H2: { t: 'n', w: '90', v: 90 },
         I2: { t: 'n', w: '90', v: 90 },
         J2: { t: 's', v: 'release' },
         K2: { t: 's', v: 'capture' },
-        A3: { z: 'm/d/yy', t: 'd', v: '2024-10-10T07:00:00.000Z', w: '10/10/24' },
+        A3: { z: 'yyyy-mm-dd', t: 'd', v: new Date('2024-10-10T07:00:00.000Z'), w: '2024-10-10' },
         B3: { t: 's', v: 'Carlita' },
         D3: { t: 'n', w: '90', v: 90 },
         E3: { t: 'n', w: '100', v: 100 },

api/src/services/import-services/marking/import-markings-strategy.test.ts

@@ -24,7 +24,7 @@ describe('ImportMarkingsStrategy', () => {
         G1: { t: 's', v: 'PRIMARY_COLOUR' },
         H1: { t: 's', v: 'SECONDARY_COLOUR' },
         I1: { t: 's', v: 'DESCRIPTION' }, // testing alias works
-        A2: { z: 'm/d/yy', t: 'd', v: '2024-10-10T07:00:00.000Z', w: '10/10/24' },
+        A2: { z: 'yyyy-mm-dd', t: 'd', v: new Date('2024-10-10T07:00:00.000Z'), w: '2024-10-10' },
         B2: { t: 's', v: 'Carl' },
         C2: { t: 's', v: '10:10:12' },
         D2: { t: 's', v: 'Left ear' }, // testing case insensitivity
@@ -100,8 +100,8 @@ describe('ImportMarkingsStrategy', () => {
       try {
         const data = await importCSV(new MediaFile('test', 'test', 'test' as unknown as Buffer), strategy);
         expect(data).to.deep.equal(2);
-      } catch (err: any) {
-        expect.fail();
+      } catch (error: any) {
+        expect.fail(error);
       }
     });
   });

api/src/services/import-services/measurement/import-measurements-strategy.test.ts

@@ -26,12 +26,12 @@ describe('importMeasurementsStrategy', () => {
         E1: { t: 's', v: 'skull condition' },
         F1: { t: 's', v: 'unknown' },
         A2: { t: 's', v: 'carl' },
-        B2: { z: 'm/d/yy', t: 'd', v: '2024-10-10T07:00:00.000Z', w: '10/10/24' },
+        B2: { t: 's', v: '2024-10-10' },
         C2: { t: 's', v: '10:10:12' },
         D2: { t: 'n', w: '2', v: 2 },
         E2: { t: 'n', w: '0', v: 'good' },
         A3: { t: 's', v: 'carlita' },
-        B3: { z: 'm/d/yy', t: 'd', v: '2024-10-10T07:00:00.000Z', w: '10/10/24' },
+        B3: { z: 'yyyy-mm-dd', t: 'd', v: new Date('2024-10-10T07:00:00.000Z'), w: '2024-10-10' },
         C3: { t: 's', v: '10:10:12' },
         D3: { t: 'n', w: '2', v: 2 },
         E3: { t: 'n', w: '0', v: 'good' },
@@ -54,7 +54,7 @@ describe('importMeasurementsStrategy', () => {
             critter_id: 'A',
             animal_id: 'carl',
             itis_tsn: 'tsn1',
-            captures: [{ capture_id: 'B', capture_date: '10/10/2024', capture_time: '10:10:12' }]
+            captures: [{ capture_id: 'B', capture_date: '2024-10-10', capture_time: '10:10:12' }]
           } as any
         ],
         [
@@ -63,7 +63,7 @@ describe('importMeasurementsStrategy', () => {
             critter_id: 'B',
             animal_id: 'carlita',
             itis_tsn: 'tsn2',
-            captures: [{ capture_id: 'B', capture_date: '10/10/2024', capture_time: '10:10:12' }]
+            captures: [{ capture_id: 'B', capture_date: '2024-10-10', capture_time: '10:10:12' }]
           } as any
         ]
       ]);
@@ -143,8 +143,8 @@ describe('importMeasurementsStrategy', () => {
             }
           ]
         });
-      } catch (e: any) {
-        expect.fail();
+      } catch (error: any) {
+        expect.fail(error);
       }
     });
   });
@@ -183,15 +183,15 @@ describe('importMeasurementsStrategy', () => {
       const conn = getMockDBConnection();
       const strategy = new ImportMeasurementsStrategy(conn, 1);
 
-      const row = { ALIAS: 'alias', CAPTURE_DATE: '10/10/2024', CAPTURE_TIME: '10:10:10' };
+      const row = { ALIAS: 'alias', CAPTURE_DATE: '2024-10-10', CAPTURE_TIME: '10:10:10' };
       const critterAliasMap = new Map([
         [
           'alias',
           {
             critter_id: 'A',
             animal_id: 'alias',
             itis_tsn: 'tsn1',
-            captures: [{ capture_id: 'B', capture_date: '10/10/2024', capture_time: '10:10:10' }]
+            captures: [{ capture_id: 'B', capture_date: '2024-10-10', capture_time: '10:10:10' }]
           } as any
         ]
       ]);
@@ -205,15 +205,15 @@ describe('importMeasurementsStrategy', () => {
       const conn = getMockDBConnection();
       const strategy = new ImportMeasurementsStrategy(conn, 1);
 
-      const row = { ALIAS: 'alias', CAPTURE_DATE: '10/10/2024', CAPTURE_TIME: '10:10:10' };
+      const row = { ALIAS: 'alias', CAPTURE_DATE: '2024-10-10', CAPTURE_TIME: '10:10:10' };
       const critterAliasMap = new Map([
         [
           'alias2',
           {
             critter_id: 'A',
             animal_id: 'alias2',
             itis_tsn: 'tsn1',
-            captures: [{ capture_id: 'B', capture_date: '10/10/2024', capture_time: '10:10:10' }]
+            captures: [{ capture_id: 'B', capture_date: '2024-10-10', capture_time: '10:10:10' }]
           } as any
         ]
       ]);
@@ -227,15 +227,15 @@ describe('importMeasurementsStrategy', () => {
       const conn = getMockDBConnection();
       const strategy = new ImportMeasurementsStrategy(conn, 1);
 
-      const row = { ALIAS: 'alias', CAPTURE_DATE: '10/10/2024', CAPTURE_TIME: '10:10:10' };
+      const row = { ALIAS: 'alias', CAPTURE_DATE: '2024-10-10', CAPTURE_TIME: '10:10:10' };
       const critterAliasMap = new Map([
         [
           'alias',
           {
             critter_id: 'A',
             animal_id: 'alias',
             itis_tsn: 'tsn1',
-            captures: [{ capture_id: 'B', capture_date: '11/11/2024', capture_time: '10:10:10' }]
+            captures: [{ capture_id: 'B', capture_date: '2024-11-11', capture_time: '10:10:10' }]
           } as any
         ]
       ]);
@@ -344,7 +344,7 @@ describe('importMeasurementsStrategy', () => {
           critter_id: 'A',
           animal_id: 'alias',
           itis_tsn: 'tsn1',
-          captures: [{ capture_id: 'B', capture_date: '10/10/2024', capture_time: '10:10:10' }]
+          captures: [{ capture_id: 'B', capture_date: '2024-10-10', capture_time: '10:10:10' }]
         } as any
       ]
     ]);
@@ -372,7 +372,7 @@ describe('importMeasurementsStrategy', () => {
       );
       validateQualitativeMeasurementCellStub.returns({ error: undefined, optionId: 'C' });
 
-      const rows = [{ ALIAS: 'alias', CAPTURE_DATE: '10/10/2024', CAPTURE_TIME: '10:10:10', measurement: 'length' }];
+      const rows = [{ ALIAS: 'alias', CAPTURE_DATE: '2024-10-10', CAPTURE_TIME: '10:10:10', measurement: 'length' }];
 
       const result = await strategy.validateRows(rows, {});
 
@@ -408,7 +408,7 @@ describe('importMeasurementsStrategy', () => {
       );
       validateQualitativeMeasurementCellStub.returns({ error: undefined, optionId: 'C' });
 
-      const rows = [{ ALIAS: 'alias', CAPTURE_DATE: '10/10/2024', CAPTURE_TIME: '10:10:10', measurement: 'length' }];
+      const rows = [{ ALIAS: 'alias', CAPTURE_DATE: '2024-10-10', CAPTURE_TIME: '10:10:10', measurement: 'length' }];
 
       const result = await strategy.validateRows(rows, {});
 
@@ -444,7 +444,7 @@ describe('importMeasurementsStrategy', () => {
       );
       validateQualitativeMeasurementCellStub.returns({ error: undefined, optionId: 'C' });
 
-      const rows = [{ ALIAS: 'alias', CAPTURE_DATE: '10/10/2024', CAPTURE_TIME: '10:10:10', measurement: 'length' }];
+      const rows = [{ ALIAS: 'alias', CAPTURE_DATE: '2024-10-10', CAPTURE_TIME: '10:10:10', measurement: 'length' }];
 
       const result = await strategy.validateRows(rows, {});
 
@@ -480,7 +480,7 @@ describe('importMeasurementsStrategy', () => {
       );
       validateQualitativeMeasurementCellStub.returns({ error: undefined, optionId: 'C' });
 
-      const rows = [{ ALIAS: 'alias', CAPTURE_DATE: '10/10/2024', CAPTURE_TIME: '10:10:10', measurement: 'length' }];
+      const rows = [{ ALIAS: 'alias', CAPTURE_DATE: '2024-10-10', CAPTURE_TIME: '10:10:10', measurement: 'length' }];
 
       const result = await strategy.validateRows(rows, {});
 
@@ -514,7 +514,7 @@ describe('importMeasurementsStrategy', () => {
       );
       validateQualitativeMeasurementCellStub.returns({ error: 'qualitative failed', optionId: undefined });
 
-      const rows = [{ ALIAS: 'alias', CAPTURE_DATE: '10/10/2024', CAPTURE_TIME: '10:10:10', measurement: 'length' }];
+      const rows = [{ ALIAS: 'alias', CAPTURE_DATE: '2024-10-10', CAPTURE_TIME: '10:10:10', measurement: 'length' }];
 
       const result = await strategy.validateRows(rows, {});
 
@@ -548,7 +548,7 @@ describe('importMeasurementsStrategy', () => {
       );
       validateQuantitativeMeasurementCellStub.returns({ error: 'quantitative failed', value: undefined });
 
-      const rows = [{ ALIAS: 'alias', CAPTURE_DATE: '10/10/2024', CAPTURE_TIME: '10:10:10', measurement: 'length' }];
+      const rows = [{ ALIAS: 'alias', CAPTURE_DATE: '2024-10-10', CAPTURE_TIME: '10:10:10', measurement: 'length' }];
 
       const result = await strategy.validateRows(rows, {});
 
@@ -573,7 +573,7 @@ describe('importMeasurementsStrategy', () => {
       getRowMetaStub.returns({ critter_id: 'A', tsn: 'tsn1', capture_id: 'C' });
       getTsnMeasurementMapStub.resolves(new Map([['tsn1', { quantitative: [], qualitative: [] } as any]]));
 
-      const rows = [{ ALIAS: 'alias', CAPTURE_DATE: '10/10/2024', CAPTURE_TIME: '10:10:10', measurement: 'length' }];
+      const rows = [{ ALIAS: 'alias', CAPTURE_DATE: '2024-10-10', CAPTURE_TIME: '10:10:10', measurement: 'length' }];
 
       const result = await strategy.validateRows(rows, {});
 
@@ -607,7 +607,7 @@ describe('importMeasurementsStrategy', () => {
         ])
       );
 
-      const rows = [{ ALIAS: 'alias', CAPTURE_DATE: '10/10/2024', CAPTURE_TIME: '10:10:10', measurement: 'length' }];
+      const rows = [{ ALIAS: 'alias', CAPTURE_DATE: '2024-10-10', CAPTURE_TIME: '10:10:10', measurement: 'length' }];
 
       const result = await strategy.validateRows(rows, {});