Skip to content

Commit

Permalink
SIMSBIOHUB-619: Import Observations With Sampling Information (#1389)
Browse files Browse the repository at this point in the history 
* New: import observations with sampling information
* Remove observation location and timestamp not null constraint
* Default sign value to direct sighting
* Improve how codes are determined from csv imports & highlight observation date bug
* Update date handling in xlsx parsing code.
* Add constants for date/time formats.
---------

Co-authored-by: Nick Phura <[email protected]>
  • Loading branch information
@mauberti-bc @NickPhura
mauberti-bc and NickPhura authored Oct 30, 2024
1 parent bb7855b commit 0d9d67d
Show file tree
Hide file tree
Showing 21 changed files with 464 additions and 104 deletions.
22 changes: 22 additions & 0 deletions api/src/constants/dates.ts
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
/*
* Date formats.
*
* See BC Gov standards: https://www2.gov.bc.ca/gov/content/governments/services-for-government/policies-procedures/web-content-development-guides/writing-for-the-web/web-style-guide/numbers
*/
export const DefaultDateFormat = 'YYYY-MM-DD'; // 2020-01-05

export const DefaultDateFormatReverse = 'DD-MM-YYYY'; // 05-01-2020

export const AltDateFormat = 'YYYY/MM/DD'; // 2020/01/05

export const AltDateFormatReverse = 'DD/MM/YYYY'; // 05/01/2020

/*
* Time formats.
*/
export const DefaultTimeFormat = 'HH:mm:ss'; // 23:00:00

/*
* Datetime formats.
*/
export const DefaultDateTimeFormat = `${DefaultDateFormat}T${DefaultTimeFormat}`; // 2020-01-05T23:00:00
2 changes: 1 addition & 1 deletion api/src/models/project-survey-attachments.ts
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import { default as dayjs } from 'dayjs';
import dayjs from 'dayjs';
import { ATTACHMENT_TYPE } from '../constants/attachments';
import { getLogger } from '../utils/logger';
import { SurveySupplementaryData } from './survey-view';
Expand Down
16 changes: 12 additions & 4 deletions api/src/openapi/schemas/observation.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,19 +60,27 @@ export const observervationsWithSubcountDataSchema: OpenAPIV3.SchemaObject = {
nullable: true
},
latitude: {
type: 'number'
type: 'number',
nullable: true,
minimum: -90,
maximum: 90
},
longitude: {
type: 'number'
type: 'number',
nullable: true,
minimum: -180,
maximum: 180
},
count: {
type: 'integer'
},
observation_date: {
type: 'string'
type: 'string',
nullable: true
},
observation_time: {
type: 'string'
type: 'string',
nullable: true
},
survey_sample_site_name: {
type: 'string',
Expand Down
9 changes: 5 additions & 4 deletions api/src/paths/project/{projectId}/survey/{surveyId}/deployments/index.ts
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import dayjs from 'dayjs';
import { RequestHandler } from 'express';
import { Operation } from 'express-openapi';
import { DefaultDateFormat, DefaultTimeFormat } from '../../../../../../constants/dates';
import { PROJECT_PERMISSION, SYSTEM_ROLE } from '../../../../../../constants/roles';
import { getDBConnection } from '../../../../../../database/db';
import { getDeploymentSchema } from '../../../../../../openapi/schemas/deployment';
Expand Down Expand Up @@ -204,16 +205,16 @@ export function getDeploymentsInSurvey(): RequestHandler {
assignment_id: matchingBctwDeployments[0].assignment_id,
collar_id: matchingBctwDeployments[0].collar_id,
attachment_start_date: matchingBctwDeployments[0].attachment_start
? dayjs(matchingBctwDeployments[0].attachment_start).format('YYYY-MM-DD')
? dayjs(matchingBctwDeployments[0].attachment_start).format(DefaultDateFormat)
: null,
attachment_start_time: matchingBctwDeployments[0].attachment_start
? dayjs(matchingBctwDeployments[0].attachment_start).format('HH:mm:ss')
? dayjs(matchingBctwDeployments[0].attachment_start).format(DefaultTimeFormat)
: null,
attachment_end_date: matchingBctwDeployments[0].attachment_end
? dayjs(matchingBctwDeployments[0].attachment_end).format('YYYY-MM-DD')
? dayjs(matchingBctwDeployments[0].attachment_end).format(DefaultDateFormat)
: null,
attachment_end_time: matchingBctwDeployments[0].attachment_end
? dayjs(matchingBctwDeployments[0].attachment_end).format('HH:mm:ss')
? dayjs(matchingBctwDeployments[0].attachment_end).format(DefaultTimeFormat)
: null,
bctw_deployment_id: matchingBctwDeployments[0].deployment_id,
device_id: matchingBctwDeployments[0].device_id,
Expand Down
9 changes: 5 additions & 4 deletions api/src/paths/project/{projectId}/survey/{surveyId}/deployments/{deploymentId}/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ import { AxiosError } from 'axios';
import dayjs from 'dayjs';
import { RequestHandler } from 'express';
import { Operation } from 'express-openapi';
import { DefaultDateFormat, DefaultTimeFormat } from '../../../../../../../constants/dates';
import { PROJECT_PERMISSION, SYSTEM_ROLE } from '../../../../../../../constants/roles';
import { getDBConnection } from '../../../../../../../database/db';
import { HTTP400 } from '../../../../../../../errors/http-error';
Expand Down Expand Up @@ -211,16 +212,16 @@ export function getDeploymentById(): RequestHandler {
assignment_id: matchingBctwDeployments[0].assignment_id,
collar_id: matchingBctwDeployments[0].collar_id,
attachment_start_date: matchingBctwDeployments[0].attachment_start
? dayjs(matchingBctwDeployments[0].attachment_start).format('YYYY-MM-DD')
? dayjs(matchingBctwDeployments[0].attachment_start).format(DefaultDateFormat)
: null,
attachment_start_time: matchingBctwDeployments[0].attachment_start
? dayjs(matchingBctwDeployments[0].attachment_start).format('HH:mm:ss')
? dayjs(matchingBctwDeployments[0].attachment_start).format(DefaultTimeFormat)
: null,
attachment_end_date: matchingBctwDeployments[0].attachment_end
? dayjs(matchingBctwDeployments[0].attachment_end).format('YYYY-MM-DD')
? dayjs(matchingBctwDeployments[0].attachment_end).format(DefaultDateFormat)
: null,
attachment_end_time: matchingBctwDeployments[0].attachment_end
? dayjs(matchingBctwDeployments[0].attachment_end).format('HH:mm:ss')
? dayjs(matchingBctwDeployments[0].attachment_end).format(DefaultTimeFormat)
: null,
bctw_deployment_id: matchingBctwDeployments[0].deployment_id,
device_id: matchingBctwDeployments[0].device_id,
Expand Down
2 changes: 1 addition & 1 deletion api/src/paths/project/{projectId}/survey/{surveyId}/observations/process.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@ POST.apiDoc = {
surveySamplePeriodId: {
type: 'integer',
description:
'The optional ID of a survey sample period to associate the parsed observation records with.'
'The optional ID of a survey sample period to associate the parsed observation records with. This is used when uploading all observations to a specific sample period, not when each record is for a different sample period.'
}
}
}
Expand Down
19 changes: 11 additions & 8 deletions api/src/repositories/observation-repository/observation-repository.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,11 +31,11 @@ export const ObservationRecord = z.object({
survey_sample_site_id: z.number().nullable(),
survey_sample_method_id: z.number().nullable(),
survey_sample_period_id: z.number().nullable(),
latitude: z.number(),
longitude: z.number(),
latitude: z.number().nullable(),
longitude: z.number().nullable(),
count: z.number(),
observation_time: z.string(),
observation_date: z.string(),
observation_time: z.string().nullable(),
observation_date: z.string().nullable(),
create_date: z.string(),
create_user: z.number(),
update_date: z.string().nullable(),
Expand Down Expand Up @@ -320,10 +320,10 @@ export class ObservationRepository extends BaseRepository {
observation.survey_sample_method_id ?? 'NULL',
observation.survey_sample_period_id ?? 'NULL',
observation.count,
observation.latitude,
observation.longitude,
`'${observation.observation_date}'`,
`'${observation.observation_time}'`,
observation.latitude ?? 'NULL',
observation.longitude ?? 'NULL',
observation.observation_date ? `'${observation.observation_date}'` : 'NULL',
observation.observation_time ? `'${observation.observation_time}'` : 'NULL',
observation.itis_tsn ?? 'NULL',
observation.itis_scientific_name ? `'${observation.itis_scientific_name}'` : 'NULL'
].join(', ')})`;
Expand Down Expand Up @@ -373,6 +373,9 @@ export class ObservationRepository extends BaseRepository {
knex.raw("JSON_BUILD_OBJECT('type', 'Point', 'coordinates', JSON_BUILD_ARRAY(longitude, latitude)) as geometry")
)
.from('survey_observation')
// TODO: For observations without lat/lon, get a location from the sampling site?
.whereNotNull('latitude')
.whereNotNull('longitude')
.where('survey_id', surveyId);

const response = await this.connection.knex(query, ObservationGeometryRecord);
Expand Down
6 changes: 3 additions & 3 deletions api/src/services/import-services/capture/import-captures-strategy.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,18 +22,18 @@ describe('import-captures-service', () => {
I1: { t: 's', v: 'RELEASE_LONGITUDE' },
J1: { t: 's', v: 'RELEASE_COMMENT' },
K1: { t: 's', v: 'CAPTURE_COMMENT' },
A2: { z: 'm/d/yy', t: 'd', v: '2024-10-10T07:00:00.000Z', w: '10/10/24' },
A2: { t: 's', v: '2024-10-11' },
B2: { t: 's', v: 'Carl' },
C2: { t: 's', v: '10:10:10' },
D2: { t: 'n', w: '90', v: 90 },
E2: { t: 'n', w: '100', v: 100 },
F2: { z: 'm/d/yy', t: 'd', v: '2024-10-10T07:00:00.000Z', w: '10/10/24' },
F2: { t: 's', v: '2024-10-10' },
G2: { t: 's', v: '9:09' },
H2: { t: 'n', w: '90', v: 90 },
I2: { t: 'n', w: '90', v: 90 },
J2: { t: 's', v: 'release' },
K2: { t: 's', v: 'capture' },
A3: { z: 'm/d/yy', t: 'd', v: '2024-10-10T07:00:00.000Z', w: '10/10/24' },
A3: { z: 'yyyy-mm-dd', t: 'd', v: new Date('2024-10-10T07:00:00.000Z'), w: '2024-10-10' },
B3: { t: 's', v: 'Carlita' },
D3: { t: 'n', w: '90', v: 90 },
E3: { t: 'n', w: '100', v: 100 },
Expand Down
6 changes: 3 additions & 3 deletions api/src/services/import-services/marking/import-markings-strategy.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ describe('ImportMarkingsStrategy', () => {
G1: { t: 's', v: 'PRIMARY_COLOUR' },
H1: { t: 's', v: 'SECONDARY_COLOUR' },
I1: { t: 's', v: 'DESCRIPTION' }, // testing alias works
A2: { z: 'm/d/yy', t: 'd', v: '2024-10-10T07:00:00.000Z', w: '10/10/24' },
A2: { z: 'yyyy-mm-dd', t: 'd', v: new Date('2024-10-10T07:00:00.000Z'), w: '2024-10-10' },
B2: { t: 's', v: 'Carl' },
C2: { t: 's', v: '10:10:12' },
D2: { t: 's', v: 'Left ear' }, // testing case insensitivity
Expand Down Expand Up @@ -100,8 +100,8 @@ describe('ImportMarkingsStrategy', () => {
try {
const data = await importCSV(new MediaFile('test', 'test', 'test' as unknown as Buffer), strategy);
expect(data).to.deep.equal(2);
} catch (err: any) {
expect.fail();
} catch (error: any) {
expect.fail(error);
}
});
});
Expand Down
42 changes: 21 additions & 21 deletions api/src/services/import-services/measurement/import-measurements-strategy.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,12 +26,12 @@ describe('importMeasurementsStrategy', () => {
E1: { t: 's', v: 'skull condition' },
F1: { t: 's', v: 'unknown' },
A2: { t: 's', v: 'carl' },
B2: { z: 'm/d/yy', t: 'd', v: '2024-10-10T07:00:00.000Z', w: '10/10/24' },
B2: { t: 's', v: '2024-10-10' },
C2: { t: 's', v: '10:10:12' },
D2: { t: 'n', w: '2', v: 2 },
E2: { t: 'n', w: '0', v: 'good' },
A3: { t: 's', v: 'carlita' },
B3: { z: 'm/d/yy', t: 'd', v: '2024-10-10T07:00:00.000Z', w: '10/10/24' },
B3: { z: 'yyyy-mm-dd', t: 'd', v: new Date('2024-10-10T07:00:00.000Z'), w: '2024-10-10' },
C3: { t: 's', v: '10:10:12' },
D3: { t: 'n', w: '2', v: 2 },
E3: { t: 'n', w: '0', v: 'good' },
Expand All @@ -54,7 +54,7 @@ describe('importMeasurementsStrategy', () => {
critter_id: 'A',
animal_id: 'carl',
itis_tsn: 'tsn1',
captures: [{ capture_id: 'B', capture_date: '10/10/2024', capture_time: '10:10:12' }]
captures: [{ capture_id: 'B', capture_date: '2024-10-10', capture_time: '10:10:12' }]
} as any
],
[
Expand All @@ -63,7 +63,7 @@ describe('importMeasurementsStrategy', () => {
critter_id: 'B',
animal_id: 'carlita',
itis_tsn: 'tsn2',
captures: [{ capture_id: 'B', capture_date: '10/10/2024', capture_time: '10:10:12' }]
captures: [{ capture_id: 'B', capture_date: '2024-10-10', capture_time: '10:10:12' }]
} as any
]
]);
Expand Down Expand Up @@ -143,8 +143,8 @@ describe('importMeasurementsStrategy', () => {
}
]
});
} catch (e: any) {
expect.fail();
} catch (error: any) {
expect.fail(error);
}
});
});
Expand Down Expand Up @@ -183,15 +183,15 @@ describe('importMeasurementsStrategy', () => {
const conn = getMockDBConnection();
const strategy = new ImportMeasurementsStrategy(conn, 1);

const row = { ALIAS: 'alias', CAPTURE_DATE: '10/10/2024', CAPTURE_TIME: '10:10:10' };
const row = { ALIAS: 'alias', CAPTURE_DATE: '2024-10-10', CAPTURE_TIME: '10:10:10' };
const critterAliasMap = new Map([
[
'alias',
{
critter_id: 'A',
animal_id: 'alias',
itis_tsn: 'tsn1',
captures: [{ capture_id: 'B', capture_date: '10/10/2024', capture_time: '10:10:10' }]
captures: [{ capture_id: 'B', capture_date: '2024-10-10', capture_time: '10:10:10' }]
} as any
]
]);
Expand All @@ -205,15 +205,15 @@ describe('importMeasurementsStrategy', () => {
const conn = getMockDBConnection();
const strategy = new ImportMeasurementsStrategy(conn, 1);

const row = { ALIAS: 'alias', CAPTURE_DATE: '10/10/2024', CAPTURE_TIME: '10:10:10' };
const row = { ALIAS: 'alias', CAPTURE_DATE: '2024-10-10', CAPTURE_TIME: '10:10:10' };
const critterAliasMap = new Map([
[
'alias2',
{
critter_id: 'A',
animal_id: 'alias2',
itis_tsn: 'tsn1',
captures: [{ capture_id: 'B', capture_date: '10/10/2024', capture_time: '10:10:10' }]
captures: [{ capture_id: 'B', capture_date: '2024-10-10', capture_time: '10:10:10' }]
} as any
]
]);
Expand All @@ -227,15 +227,15 @@ describe('importMeasurementsStrategy', () => {
const conn = getMockDBConnection();
const strategy = new ImportMeasurementsStrategy(conn, 1);

const row = { ALIAS: 'alias', CAPTURE_DATE: '10/10/2024', CAPTURE_TIME: '10:10:10' };
const row = { ALIAS: 'alias', CAPTURE_DATE: '2024-10-10', CAPTURE_TIME: '10:10:10' };
const critterAliasMap = new Map([
[
'alias',
{
critter_id: 'A',
animal_id: 'alias',
itis_tsn: 'tsn1',
captures: [{ capture_id: 'B', capture_date: '11/11/2024', capture_time: '10:10:10' }]
captures: [{ capture_id: 'B', capture_date: '2024-11-11', capture_time: '10:10:10' }]
} as any
]
]);
Expand Down Expand Up @@ -344,7 +344,7 @@ describe('importMeasurementsStrategy', () => {
critter_id: 'A',
animal_id: 'alias',
itis_tsn: 'tsn1',
captures: [{ capture_id: 'B', capture_date: '10/10/2024', capture_time: '10:10:10' }]
captures: [{ capture_id: 'B', capture_date: '2024-10-10', capture_time: '10:10:10' }]
} as any
]
]);
Expand Down Expand Up @@ -372,7 +372,7 @@ describe('importMeasurementsStrategy', () => {
);
validateQualitativeMeasurementCellStub.returns({ error: undefined, optionId: 'C' });

const rows = [{ ALIAS: 'alias', CAPTURE_DATE: '10/10/2024', CAPTURE_TIME: '10:10:10', measurement: 'length' }];
const rows = [{ ALIAS: 'alias', CAPTURE_DATE: '2024-10-10', CAPTURE_TIME: '10:10:10', measurement: 'length' }];

const result = await strategy.validateRows(rows, {});

Expand Down Expand Up @@ -408,7 +408,7 @@ describe('importMeasurementsStrategy', () => {
);
validateQualitativeMeasurementCellStub.returns({ error: undefined, optionId: 'C' });

const rows = [{ ALIAS: 'alias', CAPTURE_DATE: '10/10/2024', CAPTURE_TIME: '10:10:10', measurement: 'length' }];
const rows = [{ ALIAS: 'alias', CAPTURE_DATE: '2024-10-10', CAPTURE_TIME: '10:10:10', measurement: 'length' }];

const result = await strategy.validateRows(rows, {});

Expand Down Expand Up @@ -444,7 +444,7 @@ describe('importMeasurementsStrategy', () => {
);
validateQualitativeMeasurementCellStub.returns({ error: undefined, optionId: 'C' });

const rows = [{ ALIAS: 'alias', CAPTURE_DATE: '10/10/2024', CAPTURE_TIME: '10:10:10', measurement: 'length' }];
const rows = [{ ALIAS: 'alias', CAPTURE_DATE: '2024-10-10', CAPTURE_TIME: '10:10:10', measurement: 'length' }];

const result = await strategy.validateRows(rows, {});

Expand Down Expand Up @@ -480,7 +480,7 @@ describe('importMeasurementsStrategy', () => {
);
validateQualitativeMeasurementCellStub.returns({ error: undefined, optionId: 'C' });

const rows = [{ ALIAS: 'alias', CAPTURE_DATE: '10/10/2024', CAPTURE_TIME: '10:10:10', measurement: 'length' }];
const rows = [{ ALIAS: 'alias', CAPTURE_DATE: '2024-10-10', CAPTURE_TIME: '10:10:10', measurement: 'length' }];

const result = await strategy.validateRows(rows, {});

Expand Down Expand Up @@ -514,7 +514,7 @@ describe('importMeasurementsStrategy', () => {
);
validateQualitativeMeasurementCellStub.returns({ error: 'qualitative failed', optionId: undefined });

const rows = [{ ALIAS: 'alias', CAPTURE_DATE: '10/10/2024', CAPTURE_TIME: '10:10:10', measurement: 'length' }];
const rows = [{ ALIAS: 'alias', CAPTURE_DATE: '2024-10-10', CAPTURE_TIME: '10:10:10', measurement: 'length' }];

const result = await strategy.validateRows(rows, {});

Expand Down Expand Up @@ -548,7 +548,7 @@ describe('importMeasurementsStrategy', () => {
);
validateQuantitativeMeasurementCellStub.returns({ error: 'quantitative failed', value: undefined });

const rows = [{ ALIAS: 'alias', CAPTURE_DATE: '10/10/2024', CAPTURE_TIME: '10:10:10', measurement: 'length' }];
const rows = [{ ALIAS: 'alias', CAPTURE_DATE: '2024-10-10', CAPTURE_TIME: '10:10:10', measurement: 'length' }];

const result = await strategy.validateRows(rows, {});

Expand All @@ -573,7 +573,7 @@ describe('importMeasurementsStrategy', () => {
getRowMetaStub.returns({ critter_id: 'A', tsn: 'tsn1', capture_id: 'C' });
getTsnMeasurementMapStub.resolves(new Map([['tsn1', { quantitative: [], qualitative: [] } as any]]));

const rows = [{ ALIAS: 'alias', CAPTURE_DATE: '10/10/2024', CAPTURE_TIME: '10:10:10', measurement: 'length' }];
const rows = [{ ALIAS: 'alias', CAPTURE_DATE: '2024-10-10', CAPTURE_TIME: '10:10:10', measurement: 'length' }];

const result = await strategy.validateRows(rows, {});

Expand Down Expand Up @@ -607,7 +607,7 @@ describe('importMeasurementsStrategy', () => {
])
);

const rows = [{ ALIAS: 'alias', CAPTURE_DATE: '10/10/2024', CAPTURE_TIME: '10:10:10', measurement: 'length' }];
const rows = [{ ALIAS: 'alias', CAPTURE_DATE: '2024-10-10', CAPTURE_TIME: '10:10:10', measurement: 'length' }];

const result = await strategy.validateRows(rows, {});

Expand Down
Loading

0 comments on commit 0d9d67d

Please sign in to comment.