Skip to content

Commit

Permalink
Add Sign Field to Observation Subcounts (#1354)
Browse files Browse the repository at this point in the history 
* add subcount sign manually and by csv import

* update column name for sign

* fix tests

* remove console logs & simplify code

* Add some unit tests to new code column validator logic

* ignore-skip

* Add sign description.
Add manual observation environment column validation.
Fix sql bug preventing any quantitatitve environments from being returned in the search results.
Fix missing openapi quantitative field.
Fix frontend quantitative type.

---------

Co-authored-by: Nick Phura <[email protected]>
  • Loading branch information
@mauberti-bc @NickPhura
mauberti-bc and NickPhura authored Sep 10, 2024
1 parent 407efe1 commit 0d0f2f2
Show file tree
Hide file tree
Showing 29 changed files with 757 additions and 108 deletions.
22 changes: 20 additions & 2 deletions api/src/openapi/schemas/observation.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,7 @@ export const observervationsWithSubcountDataSchema: OpenAPIV3.SchemaObject = {
required: [
'observation_subcount_id',
'subcount',
'observation_subcount_sign_id',
'qualitative_measurements',
'quantitative_measurements',
'qualitative_environments',
Expand All @@ -103,6 +104,12 @@ export const observervationsWithSubcountDataSchema: OpenAPIV3.SchemaObject = {
observation_subcount_id: {
type: 'integer'
},
observation_subcount_sign_id: {
type: 'integer',
minimum: 1,
description:
'The observation subcount sign ID, indicating whether the subcount was a direct sighting, footprints, scat, etc.'
},
subcount: {
type: 'number'
},
Expand Down Expand Up @@ -146,7 +153,11 @@ export const observervationsWithSubcountDataSchema: OpenAPIV3.SchemaObject = {
items: {
type: 'object',
additionalProperties: false,
required: ['environment_qualitative_id', 'environment_qualitative_option_id'],
required: [
'observation_subcount_qualitative_environment_id',
'environment_qualitative_id',
'environment_qualitative_option_id'
],
properties: {
observation_subcount_qualitative_environment_id: {
type: 'integer'
Expand All @@ -167,8 +178,15 @@ export const observervationsWithSubcountDataSchema: OpenAPIV3.SchemaObject = {
items: {
type: 'object',
additionalProperties: false,
required: ['environment_quantitative_id', 'value'],
required: [
'observation_subcount_quantitative_environment_id',
'environment_quantitative_id',
'value'
],
properties: {
observation_subcount_quantitative_environment_id: {
type: 'integer'
},
environment_quantitative_id: {
type: 'string',
format: 'uuid'
Expand Down
22 changes: 22 additions & 0 deletions api/src/paths/codes.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -360,6 +360,28 @@ GET.apiDoc = {
}
}
}
},
observation_subcount_signs: {
type: 'array',
description:
'Possible observation subcount sign ids, indicating whether the subcount was a direct sighting, footprints, scat, etc.',
items: {
type: 'object',
additionalProperties: false,
required: ['id', 'name', 'description'],
properties: {
id: {
type: 'integer',
minimum: 1
},
name: {
type: 'string'
},
description: {
type: 'string'
}
}
}
}
}
}
Expand Down
2 changes: 2 additions & 0 deletions api/src/paths/observation/index.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ describe('findObservations', () => {
{
observation_subcount_id: 9,
subcount: 5,
observation_subcount_sign_id: 1,
qualitative_measurements: [],
quantitative_measurements: [],
qualitative_environments: [],
Expand Down Expand Up @@ -131,6 +132,7 @@ describe('findObservations', () => {
{
observation_subcount_id: 9,
subcount: 5,
observation_subcount_sign_id: 1,
qualitative_measurements: [],
quantitative_measurements: [],
qualitative_environments: [],
Expand Down
39 changes: 24 additions & 15 deletions api/src/paths/project/{projectId}/survey/{surveyId}/observations/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,9 @@ import { getDBConnection } from '../../../../../../database/db';
import { observervationsWithSubcountDataSchema } from '../../../../../../openapi/schemas/observation';
import { paginationRequestQueryParamSchema } from '../../../../../../openapi/schemas/pagination';
import { authorizeRequestHandler } from '../../../../../../request-handlers/security/authorization';
import { CritterbaseService, ICritterbaseUser } from '../../../../../../services/critterbase-service';
import { CritterbaseService, getCritterbaseUser } from '../../../../../../services/critterbase-service';
import { InsertUpdateObservations, ObservationService } from '../../../../../../services/observation-service';
import { ObservationSubCountEnvironmentService } from '../../../../../../services/observation-subcount-environment-service';
import { getLogger } from '../../../../../../utils/logger';
import { ensureCompletePaginationOptions, makePaginationResponse } from '../../../../../../utils/pagination';
import { ApiPaginationOptions } from '../../../../../../zod-schema/pagination';
Expand Down Expand Up @@ -223,6 +224,7 @@ PUT.apiDoc = {
additionalProperties: false,
required: [
'subcount',
'observation_subcount_sign_id',
'qualitative_measurements',
'quantitative_measurements',
'qualitative_environments',
Expand All @@ -236,6 +238,12 @@ PUT.apiDoc = {
description:
'The observation subcount ID. If provided, the mataching existing subcount record will be updated. If not provided, a new subcount record will be inserted.'
},
observation_subcount_sign_id: {
type: 'integer',
minimum: 1,
description:
'The observation subcount sign ID, indicating whether the subcount was a direct sighting, footprints, scat, etc.'
},
subcount: {
type: 'number',
description: "The subcount record's count."
Expand Down Expand Up @@ -412,39 +420,40 @@ export function getSurveyObservations(): RequestHandler {
*/
export function putObservations(): RequestHandler {
return async (req, res) => {
const surveyId = Number(req.params.surveyId);

defaultLog.debug({ label: 'insertUpdateSurveyObservations', surveyId });

const connection = getDBConnection(req.keycloak_token);

try {
const surveyId = Number(req.params.surveyId);
const observationRows: InsertUpdateObservations[] = req.body.surveyObservations;

defaultLog.debug({ label: 'insertUpdateSurveyObservations', surveyId });

await connection.open();

const observationService = new ObservationService(connection);

const observationRows: InsertUpdateObservations[] = req.body.surveyObservations;

const user: ICritterbaseUser = {
keycloak_guid: connection.systemUserGUID(),
username: connection.systemUserIdentifier()
};
// Validate measurement data against fetched measurement definition
const critterBaseService = new CritterbaseService(getCritterbaseUser(req));
const observationSubCountEnvironmentService = new ObservationSubCountEnvironmentService(connection);

const critterBaseService = new CritterbaseService(user);
const isValid = await observationService.validateSurveyObservations(
observationRows,
critterBaseService,
observationSubCountEnvironmentService
);

// Validate measurement data against fetched measurement definition
const isValid = await observationService.validateSurveyObservations(observationRows, critterBaseService);
if (!isValid) {
throw new Error('Failed to save observation data, failed data validation.');
}

// Insert/update observation records
await observationService.insertUpdateManualSurveyObservations(surveyId, observationRows);

await connection.commit();

return res.status(204).send();
} catch (error) {
defaultLog.error({ label: 'insertUpdateManualSurveyObservations', message: 'error', error });
defaultLog.error({ label: 'putObservations', message: 'error', error });
await connection.rollback();
throw error;
} finally {
Expand Down
2 changes: 1 addition & 1 deletion api/src/paths/reference/search/environment.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ import { getAPIUserDBConnection } from '../../../database/db';
import { CodeService } from '../../../services/code-service';
import { getLogger } from '../../../utils/logger';

const defaultLog = getLogger('paths/reference');
const defaultLog = getLogger('paths/reference/search/environment');

export const GET: Operation = [findSubcountEnvironments()];

Expand Down
30 changes: 28 additions & 2 deletions api/src/repositories/code-repository.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,10 @@ export const ICode = z.object({
export type ICode = z.infer<typeof ICode>;

export const CodeSet = <T extends z.ZodRawShape>(zodSchema?: T) => {
return (zodSchema && z.array(zodSchema.shape)) || z.array(ICode);
if (zodSchema) {
return z.array(z.object(zodSchema));
}
return z.array(ICode);
};

const InvestmentActionCategoryCode = ICode.extend({ agency_id: z.number() });
Expand All @@ -21,6 +24,7 @@ const SampleMethodsCode = ICode.extend({ description: z.string() });
const SurveyProgressCode = ICode.extend({ description: z.string() });
const MethodResponseMetricsCode = ICode.extend({ description: z.string() });
const AttractantCode = ICode.extend({ description: z.string() });
const ObservationSubcountSignCode = ICode.extend({ description: z.string() });

export const IAllCodeSets = z.object({
management_action_type: CodeSet(),
Expand All @@ -41,7 +45,8 @@ export const IAllCodeSets = z.object({
sample_methods: CodeSet(SampleMethodsCode.shape),
survey_progress: CodeSet(SurveyProgressCode.shape),
method_response_metrics: CodeSet(MethodResponseMetricsCode.shape),
attractants: CodeSet(AttractantCode.shape)
attractants: CodeSet(AttractantCode.shape),
observation_subcount_signs: CodeSet(ObservationSubcountSignCode.shape)
});
export type IAllCodeSets = z.infer<typeof IAllCodeSets>;

Expand Down Expand Up @@ -440,4 +445,25 @@ export class CodeRepository extends BaseRepository {

return response.rows;
}

/**
* Fetch observation subcount sign codes.
*
* @return {*}
* @memberof CodeRepository
*/
async getObservationSubcountSigns() {
const sqlStatement = SQL`
SELECT
observation_subcount_sign_id AS id,
name,
description
FROM observation_subcount_sign
WHERE record_end_date IS null;
`;

const response = await this.connection.sql(sqlStatement, ObservationSubcountSignCode);

return response.rows;
}
}
1 change: 1 addition & 0 deletions api/src/repositories/observation-repository/observation-repository.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,7 @@ const ObservationSubcountQuantitativeEnvironmentObject = ObservationSubCountQuan

const ObservationSubcountObject = z.object({
observation_subcount_id: ObservationSubCountRecord.shape.observation_subcount_id,
observation_subcount_sign_id: ObservationSubCountRecord.shape.observation_subcount_sign_id,
subcount: ObservationSubCountRecord.shape.subcount,
qualitative_measurements: z.array(ObservationSubcountQualitativeMeasurementObject),
quantitative_measurements: z.array(ObservationSubcountQuantitativeMeasurementObject),
Expand Down
3 changes: 3 additions & 0 deletions api/src/repositories/observation-repository/utils.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@ describe('Utils', () => {
from "observation_subcount_quantitative_environment" where "observation_subcount_id" in (select "observation_subcount_id" from "observation_subcount" where "survey_observation_id" in (select "survey_observation_id" from "survey_observation" where "survey_id" in (select "survey_id" from "survey" where "survey"."project_id" in (select "project"."project_id" from "project" left join "project_participation" on "project_participation"."project_id" = "project"."project_id" where "project_participation"."system_user_id" is null)))) group by "observation_subcount_id"), "w_subcounts" as (select "survey_observation_id",
json_agg(json_build_object(
'observation_subcount_id', observation_subcount.observation_subcount_id,
'observation_subcount_sign_id', observation_subcount.observation_subcount_sign_id,
'subcount', subcount,
'qualitative_measurements', COALESCE(w_qualitative_measurements.qualitative_measurements, '[]'::json),
'quantitative_measurements', COALESCE(w_quantitative_measurements.quantitative_measurements, '[]'::json),
Expand Down Expand Up @@ -110,6 +111,7 @@ describe('Utils', () => {
from "observation_subcount_quantitative_environment" where "observation_subcount_id" in (select "observation_subcount_id" from "observation_subcount" where "survey_observation_id" in (select "survey_observation_id" from "survey_observation" where "survey_id" in (select "survey_id" from "survey" where "p"."project_id" in (select "project_id" from "project_participation" where "system_user_id" = $5)))) group by "observation_subcount_id"), "w_subcounts" as (select "survey_observation_id",
json_agg(json_build_object(
'observation_subcount_id', observation_subcount.observation_subcount_id,
'observation_subcount_sign_id', observation_subcount.observation_subcount_sign_id,
'subcount', subcount,
'qualitative_measurements', COALESCE(w_qualitative_measurements.qualitative_measurements, '[]'::json),
'quantitative_measurements', COALESCE(w_quantitative_measurements.quantitative_measurements, '[]'::json),
Expand Down Expand Up @@ -179,6 +181,7 @@ describe('Utils', () => {
from "observation_subcount_quantitative_environment" where "observation_subcount_id" in (select "observation_subcount_id" from "observation_subcount" where "survey_observation_id" in (select "survey_observation_id" from "survey_observation" where "survey_id" in (select "survey_id" from "survey" where "survey_id" = $5))) group by "observation_subcount_id"), "w_subcounts" as (select "survey_observation_id",
json_agg(json_build_object(
'observation_subcount_id', observation_subcount.observation_subcount_id,
'observation_subcount_sign_id', observation_subcount.observation_subcount_sign_id,
'subcount', subcount,
'qualitative_measurements', COALESCE(w_qualitative_measurements.qualitative_measurements, '[]'::json),
'quantitative_measurements', COALESCE(w_quantitative_measurements.quantitative_measurements, '[]'::json),
Expand Down
1 change: 1 addition & 0 deletions api/src/repositories/observation-repository/utils.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -252,6 +252,7 @@ export function getSurveyObservationsBaseQuery(
knex.raw(`
json_agg(json_build_object(
'observation_subcount_id', observation_subcount.observation_subcount_id,
'observation_subcount_sign_id', observation_subcount.observation_subcount_sign_id,
'subcount', subcount,
'qualitative_measurements', COALESCE(w_qualitative_measurements.qualitative_measurements, '[]'::json),
'quantitative_measurements', COALESCE(w_quantitative_measurements.quantitative_measurements, '[]'::json),
Expand Down
Loading

0 comments on commit 0d0f2f2

Please sign in to comment.