From bb4f7dd5dfe1e507fcf9194fd6fbb54f9fed1224 Mon Sep 17 00:00:00 2001 From: Macgregor Aubertin-Young <108430771+mauberti-bc@users.noreply.github.com> Date: Wed, 4 Sep 2024 15:29:42 -0700 Subject: [PATCH 1/2] Update styling for the Add User form (#1355) * update styling of add user form * fix tests * fix: tweaked some props for edit dialog --------- Co-authored-by: Mac Deluca <99926243+MacQSL@users.noreply.github.com> Co-authored-by: Mac Deluca --- app/src/AppRouter.tsx | 4 +- app/src/components/fields/CustomTextField.tsx | 10 +- .../admin/users/ActiveUsersList.test.tsx | 20 +- .../features/admin/users/ActiveUsersList.tsx | 64 +++-- .../admin/users/AddSystemUsersForm.tsx | 253 +++++++----------- .../admin/users/ManageUsersPage.test.tsx | 18 +- 6 files changed, 178 insertions(+), 191 deletions(-) diff --git a/app/src/AppRouter.tsx b/app/src/AppRouter.tsx index 80bbebf558..5e09060113 100644 --- a/app/src/AppRouter.tsx +++ b/app/src/AppRouter.tsx @@ -83,7 +83,9 @@ const AppRouter: React.FC = () => { - + + + diff --git a/app/src/components/fields/CustomTextField.tsx b/app/src/components/fields/CustomTextField.tsx index 38744291fd..4f50ed4ae3 100644 --- a/app/src/components/fields/CustomTextField.tsx +++ b/app/src/components/fields/CustomTextField.tsx @@ -9,6 +9,13 @@ export interface ICustomTextField { * @memberof ICustomTextField */ label: string; + /** + * Placeholder for the text field + * + * @type {string} + * @memberof ICustomTextField + */ + placeholder?: string; /** * Name of the text field, typically this is used to identify the field in the formik context. * @@ -34,13 +41,14 @@ export interface ICustomTextField { const CustomTextField = (props: React.PropsWithChildren) => { const { touched, errors, values, handleChange, handleBlur } = useFormikContext(); - const { name, label, other } = props; + const { name, label, other, placeholder } = props; return ( {} + } as DataLoader +}; + const renderContainer = (props: IActiveUsersListProps) => { const authState = getMockAuthState({ base: SystemAdminAuthState }); return render( - - - + + + + + ); }; diff --git a/app/src/features/admin/users/ActiveUsersList.tsx b/app/src/features/admin/users/ActiveUsersList.tsx index d6e48eabd0..f8ef7845d7 100644 --- a/app/src/features/admin/users/ActiveUsersList.tsx +++ b/app/src/features/admin/users/ActiveUsersList.tsx @@ -16,11 +16,13 @@ import { DialogContext, ISnackbarProps } from 'contexts/dialogContext'; import { APIError } from 'hooks/api/useAxios'; import { useAuthStateContext } from 'hooks/useAuthStateContext'; import { useBiohubApi } from 'hooks/useBioHubApi'; +import { useCodesContext } from 'hooks/useContext'; import { IGetAllCodeSetsResponse } from 'interfaces/useCodesApi.interface'; import { ISystemUser } from 'interfaces/useUserApi.interface'; -import { useContext, useState } from 'react'; +import { useContext, useEffect, useState } from 'react'; import { useHistory } from 'react-router'; import { Link as RouterLink } from 'react-router-dom'; +import { getCodesName } from 'utils/Utils'; import AddSystemUsersForm, { AddSystemUsersFormInitialValues, AddSystemUsersFormYupSchema, @@ -50,6 +52,12 @@ const ActiveUsersList = (props: IActiveUsersListProps) => { const [openAddUserDialog, setOpenAddUserDialog] = useState(false); + const codesContext = useCodesContext(); + + useEffect(() => { + codesContext.codesDataLoader.load(); + }, [codesContext.codesDataLoader]); + const activeUsersColumnDefs: GridColDef[] = [ { field: 'user_identifier', @@ -279,16 +287,16 @@ const ActiveUsersList = (props: IActiveUsersListProps) => { const handleAddSystemUsersSave = async (values: IAddSystemUsersForm) => { setOpenAddUserDialog(false); + const systemUser = values.systemUser; + try { - for (const systemUser of values.systemUsers) { - await biohubApi.admin.addSystemUser( - systemUser.userIdentifier, - systemUser.identitySource, - systemUser.displayName, - systemUser.email, - systemUser.systemRole - ); - } + await biohubApi.admin.addSystemUser( + systemUser.userIdentifier, + systemUser.identitySource, + systemUser.displayName, + systemUser.email, + systemUser.systemRole + ); // Refresh users list refresh(); @@ -297,7 +305,7 @@ const ActiveUsersList = (props: IActiveUsersListProps) => { open: true, snackbarMessage: ( - {values.systemUsers.length} system {values.systemUsers.length > 1 ? 'users' : 'user'} added. + Successfully added {systemUser.displayName} ) }); @@ -307,8 +315,12 @@ const ActiveUsersList = (props: IActiveUsersListProps) => { if (apiError.status === 409) { dialogContext.setErrorDialog({ open: true, - dialogTitle: 'Failed to create users', - dialogText: 'One of the users you added already exists.', + dialogTitle: 'User already exists', + dialogText: `${systemUser.displayName} already exists as a ${getCodesName( + codesContext.codesDataLoader.data, + 'system_roles', + systemUser.systemRole + )}`, onClose: () => { dialogContext.setErrorDialog({ open: false }); }, @@ -380,21 +392,29 @@ const ActiveUsersList = (props: IActiveUsersListProps) => { { - return { value: item.id, label: item.name }; - }) || [] - } - /> + <> + + This form creates a new user that will be linked to an IDIR/BCeID when an account with a matching + username, email, and account type logs in. + + { + return { value: item.id, label: item.name }; + }) || [] + } + /> + ), initialValues: AddSystemUsersFormInitialValues, - validationSchema: AddSystemUsersFormYupSchema + validationSchema: AddSystemUsersFormYupSchema, + validateOnBlur: false }} onCancel={() => setOpenAddUserDialog(false)} onSave={(values) => { diff --git a/app/src/features/admin/users/AddSystemUsersForm.tsx b/app/src/features/admin/users/AddSystemUsersForm.tsx index 02fafd2ba8..2f13b1d813 100644 --- a/app/src/features/admin/users/AddSystemUsersForm.tsx +++ b/app/src/features/admin/users/AddSystemUsersForm.tsx @@ -1,17 +1,9 @@ -import { mdiPlus, mdiTrashCanOutline } from '@mdi/js'; -import Icon from '@mdi/react'; import Box from '@mui/material/Box'; -import Button from '@mui/material/Button'; -import FormControl from '@mui/material/FormControl'; -import FormHelperText from '@mui/material/FormHelperText'; -import Grid from '@mui/material/Grid'; -import IconButton from '@mui/material/IconButton'; -import InputLabel from '@mui/material/InputLabel'; -import MenuItem from '@mui/material/MenuItem'; -import Select from '@mui/material/Select'; +import Stack from '@mui/material/Stack'; +import AutocompleteField, { IAutocompleteFieldOption } from 'components/fields/AutocompleteField'; import CustomTextField from 'components/fields/CustomTextField'; import { SYSTEM_IDENTITY_SOURCE } from 'constants/auth'; -import { FieldArray, FieldArrayRenderProps, useFormikContext } from 'formik'; +import { useFormikContext } from 'formik'; import React from 'react'; import yup from 'utils/YupSchema'; @@ -24,7 +16,7 @@ export interface IAddSystemUsersFormArrayItem { } export interface IAddSystemUsersForm { - systemUsers: IAddSystemUsersFormArrayItem[]; + systemUser: IAddSystemUsersFormArrayItem; } export const AddSystemUsersFormArrayItemInitialValues: IAddSystemUsersFormArrayItem = { @@ -36,168 +28,107 @@ export const AddSystemUsersFormArrayItemInitialValues: IAddSystemUsersFormArrayI }; export const AddSystemUsersFormInitialValues: IAddSystemUsersForm = { - systemUsers: [AddSystemUsersFormArrayItemInitialValues] + systemUser: AddSystemUsersFormArrayItemInitialValues }; export const AddSystemUsersFormYupSchema = yup.object().shape({ - systemUsers: yup.array().of( - yup.object().shape({ - userIdentifier: yup.string().required('Username is required'), - displayName: yup.string().required('Display Name is required'), - email: yup.string().email('Must be a valid email').required('Email is required'), - identitySource: yup.string().required('Login Method is required'), - systemRole: yup.number().required('Role is required') - }) - ) + systemUser: yup.object().shape({ + userIdentifier: yup.string().required('Username is required'), + displayName: yup.string().required('Display Name is required'), + email: yup.string().email('Must be a valid email').required('Email is required'), + identitySource: yup.string().required('Account Type is required'), + systemRole: yup.number().required('System Role is required') + }) }); export interface AddSystemUsersFormProps { - systemRoles: any[]; + systemRoles: IAutocompleteFieldOption[]; } +/** + * Returns form component for manually adding system users before access is requested + * + * @param props + * @returns + */ const AddSystemUsersForm: React.FC = (props) => { - const { values, handleChange, handleSubmit, getFieldMeta } = useFormikContext(); + const { values, handleSubmit, getFieldMeta } = useFormikContext(); + + const userIdentifierMeta = getFieldMeta('systemUser.userIdentifier'); + const displayNameMeta = getFieldMeta('systemUser.displayName'); + const emailMeta = getFieldMeta('systemUser.email'); + + const { systemRoles } = props; + const identitySources: IAutocompleteFieldOption[] = [ + { value: SYSTEM_IDENTITY_SOURCE.IDIR as string, label: SYSTEM_IDENTITY_SOURCE.IDIR }, + { value: SYSTEM_IDENTITY_SOURCE.BCEID_BASIC as string, label: SYSTEM_IDENTITY_SOURCE.BCEID_BASIC }, + { value: SYSTEM_IDENTITY_SOURCE.BCEID_BUSINESS as string, label: SYSTEM_IDENTITY_SOURCE.BCEID_BUSINESS }, + { value: SYSTEM_IDENTITY_SOURCE.UNVERIFIED as string, label: SYSTEM_IDENTITY_SOURCE.UNVERIFIED } + ]; return (
- ( - - - {values.systemUsers?.map((systemUser: IAddSystemUsersFormArrayItem, index: number) => { - const userIdentifierMeta = getFieldMeta(`systemUsers.[${index}].userIdentifier`); - const displayNameMeta = getFieldMeta(`systemUsers.[${index}].displayName`); - const emailMeta = getFieldMeta(`systemUsers.[${index}].email`); - const identitySourceMeta = getFieldMeta(`systemUsers.[${index}].identitySource`); - const systemRoleMeta = getFieldMeta(`systemUsers.[${index}].roleId`); - - return ( - - - - - - - - - - - - - - - - Login Method - - - {identitySourceMeta.touched && identitySourceMeta.error} - - - - - - System Role - - - {systemRoleMeta.touched && systemRoleMeta.error} - - - - - arrayHelpers.remove(index)} - sx={{ - marginTop: '8px' - }}> - - - - - - ); - })} - - - - + + + + - )} - /> + + + + + + + + + + + + ); }; diff --git a/app/src/features/admin/users/ManageUsersPage.test.tsx b/app/src/features/admin/users/ManageUsersPage.test.tsx index 3cd33589cc..c9d44764ba 100644 --- a/app/src/features/admin/users/ManageUsersPage.test.tsx +++ b/app/src/features/admin/users/ManageUsersPage.test.tsx @@ -1,8 +1,11 @@ import { AuthStateContext } from 'contexts/authStateContext'; +import { CodesContext, ICodesContext } from 'contexts/codesContext'; import { createMemoryHistory } from 'history'; import { useBiohubApi } from 'hooks/useBioHubApi'; +import { DataLoader } from 'hooks/useDataLoader'; import { Router } from 'react-router'; import { getMockAuthState, SystemAdminAuthState } from 'test-helpers/auth-helpers'; +import { codes } from 'test-helpers/code-helpers'; import { cleanup, render, waitFor } from 'test-helpers/test-utils'; import ManageUsersPage from './ManageUsersPage'; @@ -11,11 +14,20 @@ const history = createMemoryHistory(); const renderContainer = () => { const authState = getMockAuthState({ base: SystemAdminAuthState }); + const mockCodesContext: ICodesContext = { + codesDataLoader: { + data: codes, + load: () => {} + } as DataLoader + }; + return render( - - - + + + + + ); }; From 981ee0c54cbb0048eb4da88ab6e985f00f7dd6ee Mon Sep 17 00:00:00 2001 From: Mac Deluca <99926243+MacQSL@users.noreply.github.com> Date: Wed, 4 Sep 2024 16:32:23 -0700 Subject: [PATCH 2/2] SIMSBIOHUB 587: API Measurements (#1344) - Bulk measurements endpoint --- api/.docker/api/Dockerfile | 3 + .../critters/measurements/import.ts | 161 +++++ api/src/services/critterbase-service.ts | 4 +- .../capture/import-captures-strategy.ts | 44 +- .../critter/import-critters-strategy.ts | 14 +- .../import-services/import-csv.interface.ts | 5 + .../marking/import-markings-strategy.ts | 28 +- .../import-measurements-strategy.interface.ts | 22 + .../import-measurements-strategy.test.ts | 650 ++++++++++++++++++ .../import-measurements-strategy.ts | 344 +++++++++ api/src/services/observation-service.ts | 20 +- .../xlsx-utils/column-validator-utils.test.ts | 20 +- .../xlsx-utils/column-validator-utils.ts | 27 +- api/src/utils/xlsx-utils/worksheet-utils.ts | 6 +- 14 files changed, 1272 insertions(+), 76 deletions(-) create mode 100644 api/src/paths/project/{projectId}/survey/{surveyId}/critters/measurements/import.ts create mode 100644 api/src/services/import-services/measurement/import-measurements-strategy.interface.ts create mode 100644 api/src/services/import-services/measurement/import-measurements-strategy.test.ts create mode 100644 api/src/services/import-services/measurement/import-measurements-strategy.ts diff --git a/api/.docker/api/Dockerfile b/api/.docker/api/Dockerfile index 2c658d84b8..a5052b22ce 100644 --- a/api/.docker/api/Dockerfile +++ b/api/.docker/api/Dockerfile @@ -21,6 +21,9 @@ ENV PATH ${HOME}/node_modules/.bin/:/usr/local/sbin:/usr/local/bin:/usr/sbin:/us # Copy the rest of the files COPY . ./ +# Update log directory file permissions, prevents permission errors for linux environments +RUN chmod -R a+rw data/logs/* + VOLUME ${HOME} # start api with live reload diff --git a/api/src/paths/project/{projectId}/survey/{surveyId}/critters/measurements/import.ts b/api/src/paths/project/{projectId}/survey/{surveyId}/critters/measurements/import.ts new file mode 100644 index 0000000000..9988b6bb00 --- /dev/null +++ b/api/src/paths/project/{projectId}/survey/{surveyId}/critters/measurements/import.ts @@ -0,0 +1,161 @@ +import { RequestHandler } from 'express'; +import { Operation } from 'express-openapi'; +import { PROJECT_PERMISSION, SYSTEM_ROLE } from '../../../../../../../constants/roles'; +import { getDBConnection } from '../../../../../../../database/db'; +import { HTTP400 } from '../../../../../../../errors/http-error'; +import { csvFileSchema } from '../../../../../../../openapi/schemas/file'; +import { authorizeRequestHandler } from '../../../../../../../request-handlers/security/authorization'; +import { importCSV } from '../../../../../../../services/import-services/import-csv'; +import { ImportMeasurementsStrategy } from '../../../../../../../services/import-services/measurement/import-measurements-strategy'; +import { scanFileForVirus } from '../../../../../../../utils/file-utils'; +import { getLogger } from '../../../../../../../utils/logger'; +import { parseMulterFile } from '../../../../../../../utils/media/media-utils'; +import { getFileFromRequest } from '../../../../../../../utils/request'; + +const defaultLog = getLogger('/api/project/{projectId}/survey/{surveyId}/measurements/import'); + +export const POST: Operation = [ + authorizeRequestHandler((req) => { + return { + or: [ + { + validProjectPermissions: [PROJECT_PERMISSION.COORDINATOR, PROJECT_PERMISSION.COLLABORATOR], + surveyId: Number(req.params.surveyId), + discriminator: 'ProjectPermission' + }, + { + validSystemRoles: [SYSTEM_ROLE.DATA_ADMINISTRATOR], + discriminator: 'SystemRole' + } + ] + }; + }), + importCsv() +]; + +POST.apiDoc = { + description: 'Upload Critterbase CSV Measurements file', + tags: ['observations'], + security: [ + { + Bearer: [] + } + ], + parameters: [ + { + in: 'path', + description: 'SIMS survey id', + name: 'projectId', + required: true, + schema: { + type: 'integer', + minimum: 1 + } + }, + { + in: 'path', + description: 'SIMS survey id', + name: 'surveyId', + required: true, + schema: { + type: 'integer', + minimum: 1 + } + } + ], + requestBody: { + description: 'Critterbase Measurements CSV import file.', + content: { + 'multipart/form-data': { + schema: { + type: 'object', + additionalProperties: false, + required: ['media'], + properties: { + media: { + description: 'Critterbase Measurements CSV import file.', + type: 'array', + minItems: 1, + maxItems: 1, + items: csvFileSchema + } + } + } + } + } + }, + responses: { + 201: { + description: 'Measurement import success.', + content: { + 'application/json': { + schema: { + type: 'object', + additionalProperties: false, + properties: { + measurementsCreated: { + description: 'Number of Critterbase measurements created.', + type: 'integer' + } + } + } + } + } + }, + 400: { + $ref: '#/components/responses/400' + }, + 401: { + $ref: '#/components/responses/401' + }, + 403: { + $ref: '#/components/responses/403' + }, + 500: { + $ref: '#/components/responses/500' + }, + default: { + $ref: '#/components/responses/default' + } + } +}; + +/** + * Imports a `Critterbase Measurement CSV` which bulk adds measurements to Critterbase. + * + * @return {*} {RequestHandler} + */ +export function importCsv(): RequestHandler { + return async (req, res) => { + const surveyId = Number(req.params.surveyId); + const rawFile = getFileFromRequest(req); + + const connection = getDBConnection(req.keycloak_token); + + try { + await connection.open(); + + // Check for viruses / malware + const virusScanResult = await scanFileForVirus(rawFile); + + if (!virusScanResult) { + throw new HTTP400('Malicious content detected, import cancelled.'); + } + + const importCsvMeasurementsStrategy = new ImportMeasurementsStrategy(connection, surveyId); + + // Pass CSV file and importer as dependencies + const measurementsCreated = await importCSV(parseMulterFile(rawFile), importCsvMeasurementsStrategy); + + await connection.commit(); + + return res.status(201).json({ measurementsCreated }); + } catch (error) { + defaultLog.error({ label: 'importMeasurementsCSV', message: 'error', error }); + await connection.rollback(); + throw error; + } finally { + connection.release(); + } + }; +} diff --git a/api/src/services/critterbase-service.ts b/api/src/services/critterbase-service.ts index 0a0e7485a2..f76067ad21 100644 --- a/api/src/services/critterbase-service.ts +++ b/api/src/services/critterbase-service.ts @@ -178,8 +178,8 @@ export interface IQualMeasurement { capture_id?: string; mortality_id?: string; qualitative_option_id: string; - measurement_comment: string; - measured_timestamp: string; + measurement_comment?: string; + measured_timestamp?: string; } export interface IQuantMeasurement { diff --git a/api/src/services/import-services/capture/import-captures-strategy.ts b/api/src/services/import-services/capture/import-captures-strategy.ts index e238ec59e3..f69ab1469e 100644 --- a/api/src/services/import-services/capture/import-captures-strategy.ts +++ b/api/src/services/import-services/capture/import-captures-strategy.ts @@ -2,7 +2,7 @@ import { v4 as uuid } from 'uuid'; import { z } from 'zod'; import { IDBConnection } from '../../../database/db'; import { CSV_COLUMN_ALIASES } from '../../../utils/xlsx-utils/column-aliases'; -import { generateCellGetterFromColumnValidator } from '../../../utils/xlsx-utils/column-validator-utils'; +import { generateColumnCellGetterFromColumnValidator } from '../../../utils/xlsx-utils/column-validator-utils'; import { IXLSXCSVValidator } from '../../../utils/xlsx-utils/worksheet-utils'; import { ICapture, ILocation } from '../../critterbase-service'; import { DBService } from '../../db-service'; @@ -64,7 +64,7 @@ export class ImportCapturesStrategy extends DBService implements CSVImportStrate */ async validateRows(rows: Row[]) { // Generate type-safe cell getter from column validator - const getCellValue = generateCellGetterFromColumnValidator(this.columnValidator); + const getColumnCell = generateColumnCellGetterFromColumnValidator(this.columnValidator); const critterAliasMap = await this.surveyCritterService.getSurveyCritterAliasMap(this.surveyId); const rowsToValidate = []; @@ -72,24 +72,24 @@ export class ImportCapturesStrategy extends DBService implements CSVImportStrate for (const row of rows) { let critterId, captureId; - const alias = getCellValue(row, 'ALIAS'); + const alias = getColumnCell(row, 'ALIAS'); - const releaseLatitude = getCellValue(row, 'RELEASE_LATITUDE'); - const releaseLongitude = getCellValue(row, 'RELEASE_LONGITUDE'); - const captureDate = getCellValue(row, 'CAPTURE_DATE'); - const captureTime = getCellValue(row, 'CAPTURE_TIME'); - const releaseTime = getCellValue(row, 'RELEASE_TIME'); + const releaseLatitude = getColumnCell(row, 'RELEASE_LATITUDE'); + const releaseLongitude = getColumnCell(row, 'RELEASE_LONGITUDE'); + const captureDate = getColumnCell(row, 'CAPTURE_DATE'); + const captureTime = getColumnCell(row, 'CAPTURE_TIME'); + const releaseTime = getColumnCell(row, 'RELEASE_TIME'); - const releaseLocationId = releaseLatitude && releaseLongitude ? uuid() : undefined; - const formattedCaptureTime = formatTimeString(captureTime); - const formattedReleaseTime = formatTimeString(releaseTime); + const releaseLocationId = releaseLatitude.cell && releaseLongitude.cell ? uuid() : undefined; + const formattedCaptureTime = formatTimeString(captureTime.cell); + const formattedReleaseTime = formatTimeString(releaseTime.cell); // If the alias is included attempt to retrieve the critterId from row // Checks if date time fields are unique for the critter's captures - if (alias) { - const critter = critterAliasMap.get(alias.toLowerCase()); + if (alias.cell) { + const critter = critterAliasMap.get(alias.cell.toLowerCase()); if (critter) { - const captures = findCapturesFromDateTime(critter.captures, captureDate, captureTime); + const captures = findCapturesFromDateTime(critter.captures, captureDate.cell, captureTime.cell); critterId = critter.critter_id; // Only set the captureId if a capture does not exist with matching date time captureId = captures.length > 0 ? undefined : uuid(); @@ -100,17 +100,17 @@ export class ImportCapturesStrategy extends DBService implements CSVImportStrate capture_id: captureId, // this will be undefined if capture exists with same date / time critter_id: critterId, capture_location_id: uuid(), - capture_date: captureDate, + capture_date: captureDate.cell, capture_time: formattedCaptureTime, - capture_latitude: getCellValue(row, 'CAPTURE_LATITUDE'), - capture_longitude: getCellValue(row, 'CAPTURE_LONGITUDE'), + capture_latitude: getColumnCell(row, 'CAPTURE_LATITUDE').cell, + capture_longitude: getColumnCell(row, 'CAPTURE_LONGITUDE').cell, release_location_id: releaseLocationId, - release_date: getCellValue(row, 'RELEASE_DATE'), + release_date: getColumnCell(row, 'RELEASE_DATE').cell, release_time: formattedReleaseTime, - release_latitude: getCellValue(row, 'RELEASE_LATITUDE'), - release_longitude: getCellValue(row, 'RELEASE_LONGITUDE'), - capture_comment: getCellValue(row, 'CAPTURE_COMMENT'), - release_comment: getCellValue(row, 'RELEASE_COMMENT') + release_latitude: getColumnCell(row, 'RELEASE_LATITUDE').cell, + release_longitude: getColumnCell(row, 'RELEASE_LONGITUDE').cell, + capture_comment: getColumnCell(row, 'CAPTURE_COMMENT').cell, + release_comment: getColumnCell(row, 'RELEASE_COMMENT').cell }); } diff --git a/api/src/services/import-services/critter/import-critters-strategy.ts b/api/src/services/import-services/critter/import-critters-strategy.ts index 4174fb30f4..f1695e87c0 100644 --- a/api/src/services/import-services/critter/import-critters-strategy.ts +++ b/api/src/services/import-services/critter/import-critters-strategy.ts @@ -5,7 +5,7 @@ import { IDBConnection } from '../../../database/db'; import { ApiGeneralError } from '../../../errors/api-error'; import { getLogger } from '../../../utils/logger'; import { CSV_COLUMN_ALIASES } from '../../../utils/xlsx-utils/column-aliases'; -import { generateCellGetterFromColumnValidator } from '../../../utils/xlsx-utils/column-validator-utils'; +import { generateColumnCellGetterFromColumnValidator } from '../../../utils/xlsx-utils/column-validator-utils'; import { getNonStandardColumnNamesFromWorksheet, IXLSXCSVValidator } from '../../../utils/xlsx-utils/worksheet-utils'; import { CritterbaseService, @@ -181,17 +181,17 @@ export class ImportCrittersStrategy extends DBService implements CSVImportStrate * @returns {PartialCsvCritter[]} CSV critters before validation */ _getRowsToValidate(rows: Row[], collectionUnitColumns: string[]): PartialCsvCritter[] { - const getCellValue = generateCellGetterFromColumnValidator(this.columnValidator); + const getColumnCell = generateColumnCellGetterFromColumnValidator(this.columnValidator); return rows.map((row) => { // Standard critter properties from CSV const standardCritterRow = { critter_id: uuid(), // Generate a uuid for each critter for convienence - sex: getCellValue(row, 'SEX'), - itis_tsn: getCellValue(row, 'ITIS_TSN'), - wlh_id: getCellValue(row, 'WLH_ID'), - animal_id: getCellValue(row, 'ALIAS'), - critter_comment: getCellValue(row, 'DESCRIPTION') + sex: getColumnCell(row, 'SEX').cell, + itis_tsn: getColumnCell(row, 'ITIS_TSN').cell, + wlh_id: getColumnCell(row, 'WLH_ID').cell, + animal_id: getColumnCell(row, 'ALIAS').cell, + critter_comment: getColumnCell(row, 'DESCRIPTION').cell }; // All other properties must be collection units ie: `population unit` or `herd unit` etc... diff --git a/api/src/services/import-services/import-csv.interface.ts b/api/src/services/import-services/import-csv.interface.ts index 43c2faf622..6b843ab9e9 100644 --- a/api/src/services/import-services/import-csv.interface.ts +++ b/api/src/services/import-services/import-csv.interface.ts @@ -67,6 +67,11 @@ export type ValidationError = { * */ row: number; + /** + * CSV column header + * + */ + col?: string; /** * CSV row error message * diff --git a/api/src/services/import-services/marking/import-markings-strategy.ts b/api/src/services/import-services/marking/import-markings-strategy.ts index e1fa92f78a..97dbf52237 100644 --- a/api/src/services/import-services/marking/import-markings-strategy.ts +++ b/api/src/services/import-services/marking/import-markings-strategy.ts @@ -2,7 +2,7 @@ import { z } from 'zod'; import { IDBConnection } from '../../../database/db'; import { getLogger } from '../../../utils/logger'; import { CSV_COLUMN_ALIASES } from '../../../utils/xlsx-utils/column-aliases'; -import { generateCellGetterFromColumnValidator } from '../../../utils/xlsx-utils/column-validator-utils'; +import { generateColumnCellGetterFromColumnValidator } from '../../../utils/xlsx-utils/column-validator-utils'; import { IXLSXCSVValidator } from '../../../utils/xlsx-utils/worksheet-utils'; import { IAsSelectLookup, ICritterDetailed } from '../../critterbase-service'; import { DBService } from '../../db-service'; @@ -98,7 +98,7 @@ export class ImportMarkingsStrategy extends DBService implements CSVImportStrate */ async validateRows(rows: Row[]) { // Generate type-safe cell getter from column validator - const getCellValue = generateCellGetterFromColumnValidator(this.columnValidator); + const getColumnCell = generateColumnCellGetterFromColumnValidator(this.columnValidator); // Get validation reference data const [critterAliasMap, colours, markingTypes] = await Promise.all([ @@ -116,18 +116,18 @@ export class ImportMarkingsStrategy extends DBService implements CSVImportStrate for (const row of rows) { let critterId, captureId; - const alias = getCellValue(row, 'ALIAS'); + const alias = getColumnCell(row, 'ALIAS'); // If the alias is included attempt to retrieve the critter_id and capture_id for the row - if (alias) { - const captureDate = getCellValue(row, 'CAPTURE_DATE'); - const captureTime = getCellValue(row, 'CAPTURE_TIME'); + if (alias.cell) { + const captureDate = getColumnCell(row, 'CAPTURE_DATE'); + const captureTime = getColumnCell(row, 'CAPTURE_TIME'); - const critter = critterAliasMap.get(alias.toLowerCase()); + const critter = critterAliasMap.get(alias.cell.toLowerCase()); if (critter) { // Find the capture_id from the date time columns - const captures = findCapturesFromDateTime(critter.captures, captureDate, captureTime); + const captures = findCapturesFromDateTime(critter.captures, captureDate.cell, captureTime.cell); captureId = captures.length === 1 ? captures[0].capture_id : undefined; critterId = critter.critter_id; rowCritters.push(critter); @@ -137,12 +137,12 @@ export class ImportMarkingsStrategy extends DBService implements CSVImportStrate rowsToValidate.push({ critter_id: critterId, // Found using alias capture_id: captureId, // Found using capture date and time - body_location: getCellValue(row, 'BODY_LOCATION'), - marking_type: getCellValue(row, 'MARKING_TYPE'), - identifier: getCellValue(row, 'IDENTIFIER'), - primary_colour: getCellValue(row, 'PRIMARY_COLOUR'), - secondary_colour: getCellValue(row, 'SECONDARY_COLOUR'), - comment: getCellValue(row, 'DESCRIPTION') + body_location: getColumnCell(row, 'BODY_LOCATION').cell, + marking_type: getColumnCell(row, 'MARKING_TYPE').cell, + identifier: getColumnCell(row, 'IDENTIFIER').cell, + primary_colour: getColumnCell(row, 'PRIMARY_COLOUR').cell, + secondary_colour: getColumnCell(row, 'SECONDARY_COLOUR').cell, + comment: getColumnCell(row, 'DESCRIPTION').cell }); } // Get the critter_id -> taxonBodyLocations[] Map diff --git a/api/src/services/import-services/measurement/import-measurements-strategy.interface.ts b/api/src/services/import-services/measurement/import-measurements-strategy.interface.ts new file mode 100644 index 0000000000..2cfabf887f --- /dev/null +++ b/api/src/services/import-services/measurement/import-measurements-strategy.interface.ts @@ -0,0 +1,22 @@ +import { z } from 'zod'; + +export const CsvQualitativeMeasurementSchema = z.object({ + critter_id: z.string().uuid(), + capture_id: z.string().uuid(), + taxon_measurement_id: z.string().uuid(), + qualitative_option_id: z.string().uuid() +}); + +export const CsvQuantitativeMeasurementSchema = z.object({ + critter_id: z.string().uuid(), + capture_id: z.string().uuid(), + taxon_measurement_id: z.string().uuid(), + value: z.number() +}); + +export const CsvMeasurementSchema = CsvQualitativeMeasurementSchema.or(CsvQuantitativeMeasurementSchema); + +// Zod inferred types +export type CsvMeasurement = z.infer; +export type CsvQuantitativeMeasurement = z.infer; +export type CsvQualitativeMeasurement = z.infer; diff --git a/api/src/services/import-services/measurement/import-measurements-strategy.test.ts b/api/src/services/import-services/measurement/import-measurements-strategy.test.ts new file mode 100644 index 0000000000..9503e0569c --- /dev/null +++ b/api/src/services/import-services/measurement/import-measurements-strategy.test.ts @@ -0,0 +1,650 @@ +import { expect } from 'chai'; +import sinon from 'sinon'; +import { MediaFile } from '../../../utils/media/media-file'; +import * as worksheetUtils from '../../../utils/xlsx-utils/worksheet-utils'; +import { getMockDBConnection } from '../../../__mocks__/db'; +import { + CBQualitativeMeasurementTypeDefinition, + CBQuantitativeMeasurementTypeDefinition, + IBulkCreateResponse +} from '../../critterbase-service'; +import { importCSV } from '../import-csv'; +import { ImportMeasurementsStrategy } from './import-measurements-strategy'; + +describe('importMeasurementsStrategy', () => { + describe('importCSV', () => { + beforeEach(() => { + sinon.restore(); + }); + + it('should import the csv file correctly', async () => { + const worksheet = { + A1: { t: 's', v: 'ALIAS' }, + B1: { t: 's', v: 'CAPTURE_DATE' }, + C1: { t: 's', v: 'CAPTURE_TIME' }, + D1: { t: 's', v: 'tail length' }, + E1: { t: 's', v: 'skull condition' }, + F1: { t: 's', v: 'unknown' }, + A2: { t: 's', v: 'carl' }, + B2: { z: 'm/d/yy', t: 'd', v: '2024-10-10T07:00:00.000Z', w: '10/10/24' }, + C2: { t: 's', v: '10:10:12' }, + D2: { t: 'n', w: '2', v: 2 }, + E2: { t: 'n', w: '0', v: 'good' }, + A3: { t: 's', v: 'carlita' }, + B3: { z: 'm/d/yy', t: 'd', v: '2024-10-10T07:00:00.000Z', w: '10/10/24' }, + C3: { t: 's', v: '10:10:12' }, + D3: { t: 'n', w: '2', v: 2 }, + E3: { t: 'n', w: '0', v: 'good' }, + '!ref': 'A1:F3' + }; + + const conn = getMockDBConnection(); + const strategy = new ImportMeasurementsStrategy(conn, 1); + + const getDefaultWorksheetStub = sinon.stub(worksheetUtils, 'getDefaultWorksheet'); + const critterbaseInsertStub = sinon.stub(strategy.surveyCritterService.critterbaseService, 'bulkCreate'); + const nonStandardColumnsStub = sinon.stub(strategy, '_getNonStandardColumns'); + const critterAliasMapStub = sinon.stub(strategy.surveyCritterService, 'getSurveyCritterAliasMap'); + const getTsnMeasurementMapStub = sinon.stub(strategy, '_getTsnsMeasurementMap'); + + const critterAliasMap = new Map([ + [ + 'carl', + { + critter_id: 'A', + animal_id: 'carl', + itis_tsn: 'tsn1', + captures: [{ capture_id: 'B', capture_date: '10/10/2024', capture_time: '10:10:12' }] + } as any + ], + [ + 'carlita', + { + critter_id: 'B', + animal_id: 'carlita', + itis_tsn: 'tsn2', + captures: [{ capture_id: 'B', capture_date: '10/10/2024', capture_time: '10:10:12' }] + } as any + ] + ]); + + getDefaultWorksheetStub.returns(worksheet); + nonStandardColumnsStub.returns(['TAIL LENGTH', 'SKULL CONDITION']); + critterAliasMapStub.resolves(critterAliasMap); + critterbaseInsertStub.resolves({ + created: { qualitative_measurements: 1, quantitative_measurements: 1 } + } as IBulkCreateResponse); + getTsnMeasurementMapStub.resolves( + new Map([ + [ + 'tsn1', + { + qualitative: [ + { + taxon_measurement_id: 'Z', + measurement_name: 'skull condition', + options: [{ qualitative_option_id: 'C', option_label: 'good' }] + } + ], + quantitative: [ + { taxon_measurement_id: 'Z', measurement_name: 'tail length', min_value: 0, max_value: 10 } + ] + } as any + ], + [ + 'tsn2', + { + qualitative: [ + { + taxon_measurement_id: 'Z', + measurement_name: 'skull condition', + + options: [{ qualitative_option_id: 'C', option_label: 'good' }] + } + ], + quantitative: [ + { taxon_measurement_id: 'Z', measurement_name: 'tail length', min_value: 0, max_value: 10 } + ] + } as any + ] + ]) + ); + + try { + const data = await importCSV(new MediaFile('test', 'test', 'test' as unknown as Buffer), strategy); + expect(data).to.be.eql(2); + expect(critterbaseInsertStub).to.have.been.calledOnceWithExactly({ + qualitative_measurements: [ + { + critter_id: 'A', + capture_id: 'B', + taxon_measurement_id: 'Z', + qualitative_option_id: 'C' + }, + { + critter_id: 'B', + capture_id: 'B', + taxon_measurement_id: 'Z', + qualitative_option_id: 'C' + } + ], + quantitative_measurements: [ + { + critter_id: 'A', + capture_id: 'B', + taxon_measurement_id: 'Z', + value: 2 + }, + { + critter_id: 'B', + capture_id: 'B', + taxon_measurement_id: 'Z', + value: 2 + } + ] + }); + } catch (e: any) { + expect.fail(); + } + }); + }); + describe('_getTsnsMeasurementMap', () => { + it('should return correct taxon measurement mapping', async () => { + const conn = getMockDBConnection(); + const strategy = new ImportMeasurementsStrategy(conn, 1); + + const getTaxonMeasurementsStub = sinon.stub( + strategy.surveyCritterService.critterbaseService, + 'getTaxonMeasurements' + ); + + const measurementA: any = { qualitative: [{ tsn: 'tsn1', measurement: 'measurement1' }], quantitative: [] }; + const measurementB: any = { quantitative: [{ tsn: 'tsn2', measurement: 'measurement2', qualitative: [] }] }; + + getTaxonMeasurementsStub.onCall(0).resolves(measurementA); + + getTaxonMeasurementsStub.onCall(1).resolves(measurementB); + + const tsns = ['tsn1', 'tsn2', 'tsn2']; + + const result = await strategy._getTsnsMeasurementMap(tsns); + + const expectedResult = new Map([ + ['tsn1', measurementA], + ['tsn2', measurementB] + ]); + + expect(result).to.be.deep.equal(expectedResult); + }); + }); + + describe('_getRowMeta', () => { + it('should return correct row meta', () => { + const conn = getMockDBConnection(); + const strategy = new ImportMeasurementsStrategy(conn, 1); + + const row = { ALIAS: 'alias', CAPTURE_DATE: '10/10/2024', CAPTURE_TIME: '10:10:10' }; + const critterAliasMap = new Map([ + [ + 'alias', + { + critter_id: 'A', + animal_id: 'alias', + itis_tsn: 'tsn1', + captures: [{ capture_id: 'B', capture_date: '10/10/2024', capture_time: '10:10:10' }] + } as any + ] + ]); + + const result = strategy._getRowMeta(row, critterAliasMap); + + expect(result).to.be.deep.equal({ critter_id: 'A', tsn: 'tsn1', capture_id: 'B' }); + }); + + it('should return all undefined properties if unable to match critter', () => { + const conn = getMockDBConnection(); + const strategy = new ImportMeasurementsStrategy(conn, 1); + + const row = { ALIAS: 'alias', CAPTURE_DATE: '10/10/2024', CAPTURE_TIME: '10:10:10' }; + const critterAliasMap = new Map([ + [ + 'alias2', + { + critter_id: 'A', + animal_id: 'alias2', + itis_tsn: 'tsn1', + captures: [{ capture_id: 'B', capture_date: '10/10/2024', capture_time: '10:10:10' }] + } as any + ] + ]); + + const result = strategy._getRowMeta(row, critterAliasMap); + + expect(result).to.be.deep.equal({ critter_id: undefined, tsn: undefined, capture_id: undefined }); + }); + + it('should undefined capture_id if unable to match timestamps', () => { + const conn = getMockDBConnection(); + const strategy = new ImportMeasurementsStrategy(conn, 1); + + const row = { ALIAS: 'alias', CAPTURE_DATE: '10/10/2024', CAPTURE_TIME: '10:10:10' }; + const critterAliasMap = new Map([ + [ + 'alias', + { + critter_id: 'A', + animal_id: 'alias', + itis_tsn: 'tsn1', + captures: [{ capture_id: 'B', capture_date: '11/11/2024', capture_time: '10:10:10' }] + } as any + ] + ]); + + const result = strategy._getRowMeta(row, critterAliasMap); + + expect(result).to.be.deep.equal({ critter_id: 'A', tsn: 'tsn1', capture_id: undefined }); + }); + }); + + describe('_validateQualitativeMeasurementCell', () => { + it('should return option_id when valid', async () => { + const conn = getMockDBConnection(); + const strategy = new ImportMeasurementsStrategy(conn, 1); + + const measurement: CBQualitativeMeasurementTypeDefinition = { + itis_tsn: 1, + taxon_measurement_id: 'A', + measurement_name: 'measurement', + measurement_desc: null, + options: [{ qualitative_option_id: 'C', option_label: 'measurement', option_value: 0, option_desc: 'desc' }] + }; + + const result = strategy._validateQualitativeMeasurementCell('measurement', measurement); + + expect(result.error).to.be.undefined; + expect(result.optionId).to.be.equal('C'); + }); + + it('should return error when invalid value', async () => { + const conn = getMockDBConnection(); + const strategy = new ImportMeasurementsStrategy(conn, 1); + + const measurement: CBQualitativeMeasurementTypeDefinition = { + itis_tsn: 1, + taxon_measurement_id: 'A', + measurement_name: 'measurement', + measurement_desc: null, + options: [{ qualitative_option_id: 'C', option_label: 'measurement', option_value: 0, option_desc: 'desc' }] + }; + + const result = strategy._validateQualitativeMeasurementCell('bad', measurement); + + expect(result.error).to.exist; + expect(result.optionId).to.be.undefined; + }); + }); + + describe('_validateQuantitativeMeasurementCell', () => { + it('should return value when valid', async () => { + const conn = getMockDBConnection(); + const strategy = new ImportMeasurementsStrategy(conn, 1); + + const measurement: CBQuantitativeMeasurementTypeDefinition = { + itis_tsn: 1, + taxon_measurement_id: 'A', + measurement_name: 'measurement', + unit: 'centimeter', + min_value: 0, + max_value: 10, + measurement_desc: null + }; + + const resultA = strategy._validateQuantitativeMeasurementCell(0, measurement); + + expect(resultA.error).to.be.undefined; + expect(resultA.value).to.be.equal(0); + + const resultB = strategy._validateQuantitativeMeasurementCell(10, measurement); + + expect(resultB.error).to.be.undefined; + expect(resultB.value).to.be.equal(10); + }); + + it('should return error when invalid value', async () => { + const conn = getMockDBConnection(); + const strategy = new ImportMeasurementsStrategy(conn, 1); + + const measurement: CBQuantitativeMeasurementTypeDefinition = { + itis_tsn: 1, + taxon_measurement_id: 'A', + measurement_name: 'measurement', + unit: 'centimeter', + min_value: 0, + max_value: 10, + measurement_desc: null + }; + + const resultA = strategy._validateQuantitativeMeasurementCell(-1, measurement); + + expect(resultA.error).to.exist; + expect(resultA.value).to.be.undefined; + + const resultB = strategy._validateQuantitativeMeasurementCell(11, measurement); + + expect(resultB.error).to.exist; + expect(resultB.value).to.be.undefined; + }); + }); + + describe('_validateMeasurementCell', () => { + const critterAliasMap = new Map([ + [ + 'alias', + { + critter_id: 'A', + animal_id: 'alias', + itis_tsn: 'tsn1', + captures: [{ capture_id: 'B', capture_date: '10/10/2024', capture_time: '10:10:10' }] + } as any + ] + ]); + + it('should return no errors and data when valid rows', async () => { + const conn = getMockDBConnection(); + const strategy = new ImportMeasurementsStrategy(conn, 1); + + const nonStandardColumnsStub = sinon.stub(strategy, '_getNonStandardColumns'); + const critterAliasMapStub = sinon.stub(strategy.surveyCritterService, 'getSurveyCritterAliasMap'); + const getRowMetaStub = sinon.stub(strategy, '_getRowMeta'); + const getTsnMeasurementMapStub = sinon.stub(strategy, '_getTsnsMeasurementMap'); + const validateQualitativeMeasurementCellStub = sinon.stub(strategy, '_validateQualitativeMeasurementCell'); + + nonStandardColumnsStub.returns(['MEASUREMENT']); + critterAliasMapStub.resolves(critterAliasMap); + getRowMetaStub.returns({ critter_id: 'A', tsn: 'tsn1', capture_id: 'B' }); + getTsnMeasurementMapStub.resolves( + new Map([ + [ + 'tsn1', + { qualitative: [{ taxon_measurement_id: 'Z', measurement_name: 'measurement' }], quantitative: [] } as any + ] + ]) + ); + validateQualitativeMeasurementCellStub.returns({ error: undefined, optionId: 'C' }); + + const rows = [{ ALIAS: 'alias', CAPTURE_DATE: '10/10/2024', CAPTURE_TIME: '10:10:10', measurement: 'length' }]; + + const result = await strategy.validateRows(rows, {}); + + if (!result.success) { + expect.fail(); + } else { + expect(result.data).to.be.deep.equal([ + { critter_id: 'A', capture_id: 'B', taxon_measurement_id: 'Z', qualitative_option_id: 'C' } + ]); + } + }); + + it('should return error when unable to map alias to critter', async () => { + const conn = getMockDBConnection(); + const strategy = new ImportMeasurementsStrategy(conn, 1); + + const nonStandardColumnsStub = sinon.stub(strategy, '_getNonStandardColumns'); + const critterAliasMapStub = sinon.stub(strategy.surveyCritterService, 'getSurveyCritterAliasMap'); + const getRowMetaStub = sinon.stub(strategy, '_getRowMeta'); + const getTsnMeasurementMapStub = sinon.stub(strategy, '_getTsnsMeasurementMap'); + const validateQualitativeMeasurementCellStub = sinon.stub(strategy, '_validateQualitativeMeasurementCell'); + + nonStandardColumnsStub.returns(['MEASUREMENT']); + critterAliasMapStub.resolves(critterAliasMap); + getRowMetaStub.returns({ critter_id: undefined, tsn: undefined, capture_id: undefined }); + getTsnMeasurementMapStub.resolves( + new Map([ + [ + 'tsn1', + { qualitative: [{ taxon_measurement_id: 'Z', measurement_name: 'measurement' }], quantitative: [] } as any + ] + ]) + ); + validateQualitativeMeasurementCellStub.returns({ error: undefined, optionId: 'C' }); + + const rows = [{ ALIAS: 'alias', CAPTURE_DATE: '10/10/2024', CAPTURE_TIME: '10:10:10', measurement: 'length' }]; + + const result = await strategy.validateRows(rows, {}); + + if (!result.success) { + expect(result.error.issues).to.be.deep.equal([ + { row: 0, message: 'Unable to find matching Critter with alias.' } + ]); + } else { + expect.fail(); + } + }); + + it('should return error when unable to map capture to critter', async () => { + const conn = getMockDBConnection(); + const strategy = new ImportMeasurementsStrategy(conn, 1); + + const nonStandardColumnsStub = sinon.stub(strategy, '_getNonStandardColumns'); + const critterAliasMapStub = sinon.stub(strategy.surveyCritterService, 'getSurveyCritterAliasMap'); + const getRowMetaStub = sinon.stub(strategy, '_getRowMeta'); + const getTsnMeasurementMapStub = sinon.stub(strategy, '_getTsnsMeasurementMap'); + const validateQualitativeMeasurementCellStub = sinon.stub(strategy, '_validateQualitativeMeasurementCell'); + + nonStandardColumnsStub.returns(['MEASUREMENT']); + critterAliasMapStub.resolves(critterAliasMap); + getRowMetaStub.returns({ critter_id: 'A', tsn: 'tsn1', capture_id: undefined }); + getTsnMeasurementMapStub.resolves( + new Map([ + [ + 'tsn1', + { qualitative: [{ taxon_measurement_id: 'Z', measurement_name: 'measurement' }], quantitative: [] } as any + ] + ]) + ); + validateQualitativeMeasurementCellStub.returns({ error: undefined, optionId: 'C' }); + + const rows = [{ ALIAS: 'alias', CAPTURE_DATE: '10/10/2024', CAPTURE_TIME: '10:10:10', measurement: 'length' }]; + + const result = await strategy.validateRows(rows, {}); + + if (!result.success) { + expect(result.error.issues).to.be.deep.equal([ + { row: 0, message: 'Unable to find matching Capture with date and time.' } + ]); + } else { + expect.fail(); + } + }); + + it('should return error when unable to map tsn to critter', async () => { + const conn = getMockDBConnection(); + const strategy = new ImportMeasurementsStrategy(conn, 1); + + const nonStandardColumnsStub = sinon.stub(strategy, '_getNonStandardColumns'); + const critterAliasMapStub = sinon.stub(strategy.surveyCritterService, 'getSurveyCritterAliasMap'); + const getRowMetaStub = sinon.stub(strategy, '_getRowMeta'); + const getTsnMeasurementMapStub = sinon.stub(strategy, '_getTsnsMeasurementMap'); + const validateQualitativeMeasurementCellStub = sinon.stub(strategy, '_validateQualitativeMeasurementCell'); + + nonStandardColumnsStub.returns(['MEASUREMENT']); + critterAliasMapStub.resolves(critterAliasMap); + getRowMetaStub.returns({ critter_id: 'A', tsn: undefined, capture_id: 'C' }); + getTsnMeasurementMapStub.resolves( + new Map([ + [ + 'tsn1', + { qualitative: [{ taxon_measurement_id: 'Z', measurement_name: 'measurement' }], quantitative: [] } as any + ] + ]) + ); + validateQualitativeMeasurementCellStub.returns({ error: undefined, optionId: 'C' }); + + const rows = [{ ALIAS: 'alias', CAPTURE_DATE: '10/10/2024', CAPTURE_TIME: '10:10:10', measurement: 'length' }]; + + const result = await strategy.validateRows(rows, {}); + + if (!result.success) { + expect(result.error.issues).to.be.deep.equal([{ row: 0, message: 'Unable to find ITIS TSN for Critter.' }]); + } else { + expect.fail(); + } + }); + + it('should return error when qualitative measurement validation fails', async () => { + const conn = getMockDBConnection(); + const strategy = new ImportMeasurementsStrategy(conn, 1); + + const nonStandardColumnsStub = sinon.stub(strategy, '_getNonStandardColumns'); + const critterAliasMapStub = sinon.stub(strategy.surveyCritterService, 'getSurveyCritterAliasMap'); + const getRowMetaStub = sinon.stub(strategy, '_getRowMeta'); + const getTsnMeasurementMapStub = sinon.stub(strategy, '_getTsnsMeasurementMap'); + const validateQualitativeMeasurementCellStub = sinon.stub(strategy, '_validateQualitativeMeasurementCell'); + + nonStandardColumnsStub.returns(['MEASUREMENT']); + critterAliasMapStub.resolves(critterAliasMap); + getRowMetaStub.returns({ critter_id: 'A', tsn: 'tsn1', capture_id: 'C' }); + getTsnMeasurementMapStub.resolves( + new Map([ + [ + 'tsn1', + { qualitative: [{ taxon_measurement_id: 'Z', measurement_name: 'measurement' }], quantitative: [] } as any + ] + ]) + ); + validateQualitativeMeasurementCellStub.returns({ error: 'qualitative failed', optionId: undefined }); + + const rows = [{ ALIAS: 'alias', CAPTURE_DATE: '10/10/2024', CAPTURE_TIME: '10:10:10', measurement: 'length' }]; + + const result = await strategy.validateRows(rows, {}); + + if (!result.success) { + expect(result.error.issues).to.be.deep.equal([{ row: 0, col: 'MEASUREMENT', message: 'qualitative failed' }]); + } else { + expect.fail(); + } + }); + + it('should return error when quantitative measurement validation fails', async () => { + const conn = getMockDBConnection(); + const strategy = new ImportMeasurementsStrategy(conn, 1); + + const nonStandardColumnsStub = sinon.stub(strategy, '_getNonStandardColumns'); + const critterAliasMapStub = sinon.stub(strategy.surveyCritterService, 'getSurveyCritterAliasMap'); + const getRowMetaStub = sinon.stub(strategy, '_getRowMeta'); + const getTsnMeasurementMapStub = sinon.stub(strategy, '_getTsnsMeasurementMap'); + const validateQuantitativeMeasurementCellStub = sinon.stub(strategy, '_validateQuantitativeMeasurementCell'); + + nonStandardColumnsStub.returns(['MEASUREMENT']); + critterAliasMapStub.resolves(critterAliasMap); + getRowMetaStub.returns({ critter_id: 'A', tsn: 'tsn1', capture_id: 'C' }); + getTsnMeasurementMapStub.resolves( + new Map([ + [ + 'tsn1', + { quantitative: [{ taxon_measurement_id: 'Z', measurement_name: 'measurement' }], qualitative: [] } as any + ] + ]) + ); + validateQuantitativeMeasurementCellStub.returns({ error: 'quantitative failed', value: undefined }); + + const rows = [{ ALIAS: 'alias', CAPTURE_DATE: '10/10/2024', CAPTURE_TIME: '10:10:10', measurement: 'length' }]; + + const result = await strategy.validateRows(rows, {}); + + if (!result.success) { + expect(result.error.issues).to.be.deep.equal([{ row: 0, col: 'MEASUREMENT', message: 'quantitative failed' }]); + } else { + expect.fail(); + } + }); + + it('should return error when no measurements exist for taxon', async () => { + const conn = getMockDBConnection(); + const strategy = new ImportMeasurementsStrategy(conn, 1); + + const nonStandardColumnsStub = sinon.stub(strategy, '_getNonStandardColumns'); + const critterAliasMapStub = sinon.stub(strategy.surveyCritterService, 'getSurveyCritterAliasMap'); + const getRowMetaStub = sinon.stub(strategy, '_getRowMeta'); + const getTsnMeasurementMapStub = sinon.stub(strategy, '_getTsnsMeasurementMap'); + + nonStandardColumnsStub.returns(['MEASUREMENT']); + critterAliasMapStub.resolves(critterAliasMap); + getRowMetaStub.returns({ critter_id: 'A', tsn: 'tsn1', capture_id: 'C' }); + getTsnMeasurementMapStub.resolves(new Map([['tsn1', { quantitative: [], qualitative: [] } as any]])); + + const rows = [{ ALIAS: 'alias', CAPTURE_DATE: '10/10/2024', CAPTURE_TIME: '10:10:10', measurement: 'length' }]; + + const result = await strategy.validateRows(rows, {}); + + if (!result.success) { + expect(result.error.issues).to.be.deep.equal([ + { row: 0, col: 'MEASUREMENT', message: 'No measurements exist for this taxon.' } + ]); + } else { + expect.fail(); + } + }); + + it('should return error when no measurements exist for taxon', async () => { + const conn = getMockDBConnection(); + const strategy = new ImportMeasurementsStrategy(conn, 1); + + const nonStandardColumnsStub = sinon.stub(strategy, '_getNonStandardColumns'); + const critterAliasMapStub = sinon.stub(strategy.surveyCritterService, 'getSurveyCritterAliasMap'); + const getRowMetaStub = sinon.stub(strategy, '_getRowMeta'); + const getTsnMeasurementMapStub = sinon.stub(strategy, '_getTsnsMeasurementMap'); + + nonStandardColumnsStub.returns(['MEASUREMENT']); + critterAliasMapStub.resolves(critterAliasMap); + getRowMetaStub.returns({ critter_id: 'A', tsn: 'tsn1', capture_id: 'C' }); + getTsnMeasurementMapStub.resolves( + new Map([ + [ + 'tsn1', + { quantitative: [{ measurement_name: 'notfound' }], qualitative: [{ measurement_name: 'notfound' }] } as any + ] + ]) + ); + + const rows = [{ ALIAS: 'alias', CAPTURE_DATE: '10/10/2024', CAPTURE_TIME: '10:10:10', measurement: 'length' }]; + + const result = await strategy.validateRows(rows, {}); + + if (!result.success) { + expect(result.error.issues).to.be.deep.equal([ + { row: 0, col: 'MEASUREMENT', message: 'Unable to match column name to an existing measurement.' } + ]); + } else { + expect.fail(); + } + }); + }); + describe('insert', () => { + it('should correctly format the insert payload for critterbase bulk insert', async () => { + const conn = getMockDBConnection(); + const strategy = new ImportMeasurementsStrategy(conn, 1); + + const bulkCreateStub = sinon.stub(strategy.surveyCritterService.critterbaseService, 'bulkCreate'); + + const rows = [ + { critter_id: 'A', capture_id: 'B', taxon_measurement_id: 'C', qualitative_option_id: 'D' }, + { critter_id: 'E', capture_id: 'F', taxon_measurement_id: 'G', value: 0 } + ]; + + bulkCreateStub.resolves({ + created: { qualitative_measurements: 1, quantitative_measurements: 1 } + } as IBulkCreateResponse); + + const result = await strategy.insert(rows); + + expect(bulkCreateStub).to.have.been.calledOnceWithExactly({ + qualitative_measurements: [ + { critter_id: 'A', capture_id: 'B', taxon_measurement_id: 'C', qualitative_option_id: 'D' } + ], + quantitative_measurements: [{ critter_id: 'E', capture_id: 'F', taxon_measurement_id: 'G', value: 0 }] + }); + expect(result).to.be.eql(2); + }); + }); +}); diff --git a/api/src/services/import-services/measurement/import-measurements-strategy.ts b/api/src/services/import-services/measurement/import-measurements-strategy.ts new file mode 100644 index 0000000000..d39e69f86c --- /dev/null +++ b/api/src/services/import-services/measurement/import-measurements-strategy.ts @@ -0,0 +1,344 @@ +import { uniq } from 'lodash'; +import { WorkSheet } from 'xlsx'; +import { IDBConnection } from '../../../database/db'; +import { getLogger } from '../../../utils/logger'; +import { CSV_COLUMN_ALIASES } from '../../../utils/xlsx-utils/column-aliases'; +import { generateColumnCellGetterFromColumnValidator } from '../../../utils/xlsx-utils/column-validator-utils'; +import { getNonStandardColumnNamesFromWorksheet, IXLSXCSVValidator } from '../../../utils/xlsx-utils/worksheet-utils'; +import { + CBQualitativeMeasurementTypeDefinition, + CBQuantitativeMeasurementTypeDefinition, + ICritterDetailed +} from '../../critterbase-service'; +import { DBService } from '../../db-service'; +import { SurveyCritterService } from '../../survey-critter-service'; +import { CSVImportStrategy, Row, Validation, ValidationError } from '../import-csv.interface'; +import { findCapturesFromDateTime } from '../utils/datetime'; +import { + CsvMeasurement, + CsvQualitativeMeasurement, + CsvQuantitativeMeasurement +} from './import-measurements-strategy.interface'; + +const defaultLog = getLogger('services/import/import-measurements-strategy'); + +/** + * + * ImportMeasurementsStrategy - Injected into importCSV as the CSV import dependency + * + * @example new CSVImport(new ImportMeasurementsStrategy(connection, surveyId)).import(file); + * + * @class ImportMeasurementsStrategy + * @extends DBService + * + */ +export class ImportMeasurementsStrategy extends DBService implements CSVImportStrategy { + surveyCritterService: SurveyCritterService; + + surveyId: number; + + /** + * An XLSX validation config for the standard columns of a Measurement CSV. + * + * Note: `satisfies` allows `keyof` to correctly infer key types, while also + * enforcing uppercase object keys. + */ + columnValidator = { + ALIAS: { type: 'string', aliases: CSV_COLUMN_ALIASES.ALIAS }, + CAPTURE_DATE: { type: 'date' }, + CAPTURE_TIME: { type: 'string', optional: true } + } satisfies IXLSXCSVValidator; + + /** + * Instantiates an instance of ImportMeasurementsStrategy + * + * @param {IDBConnection} connection - Database connection + * @param {number} surveyId - Survey identifier + */ + constructor(connection: IDBConnection, surveyId: number) { + super(connection); + + this.surveyId = surveyId; + + this.surveyCritterService = new SurveyCritterService(connection); + } + + /** + * Get non-standard columns (measurement columns) from worksheet. + * + * @param {WorkSheet} worksheet - Xlsx worksheet + * @returns {string[]} Array of non-standard headers from CSV (worksheet) + */ + _getNonStandardColumns(worksheet: WorkSheet) { + return uniq(getNonStandardColumnNamesFromWorksheet(worksheet, this.columnValidator)); + } + + /** + * Get TSN measurement map for validation. + * + * For a list of TSNS return all measurements inherited or directly assigned. + * + * @async + * @param {string[]} tsns - List of ITIS TSN's + * @returns {*} + */ + async _getTsnsMeasurementMap(tsns: string[]) { + const tsnMeasurementMap = new Map< + string, + { + qualitative: CBQualitativeMeasurementTypeDefinition[]; + quantitative: CBQuantitativeMeasurementTypeDefinition[]; + } + >(); + + const uniqueTsns = [...new Set(tsns)]; + + const measurements = await Promise.all( + uniqueTsns.map((tsn) => this.surveyCritterService.critterbaseService.getTaxonMeasurements(tsn)) + ); + + uniqueTsns.forEach((tsn, index) => { + tsnMeasurementMap.set(tsn, measurements[index]); + }); + + return tsnMeasurementMap; + } + + /** + * Get row meta data for validation. + * + * @param {Row} row - CSV row + * @param {Map} critterAliasMap - Survey critter alias mapping + * @returns {{ capture_id?: string; critter_id?: string; tsn?: string }} + */ + _getRowMeta( + row: Row, + critterAliasMap: Map + ): { capture_id?: string; critter_id?: string; tsn?: string } { + const getColumnCell = generateColumnCellGetterFromColumnValidator(this.columnValidator); + + const alias = getColumnCell(row, 'ALIAS'); + const captureDate = getColumnCell(row, 'CAPTURE_DATE'); + const captureTime = getColumnCell(row, 'CAPTURE_TIME'); + + let capture_id, critter_id, tsn; + + if (alias.cell) { + const critter = critterAliasMap.get(alias.cell.toLowerCase()); + + if (critter) { + const captures = findCapturesFromDateTime(critter.captures, captureDate.cell, captureTime.cell); + critter_id = critter.critter_id; + capture_id = captures.length === 1 ? captures[0].capture_id : undefined; + tsn = String(critter.itis_tsn); // Cast to string for convienience + } + } + + return { critter_id, capture_id, tsn }; + } + + /** + * Validate qualitative measurement. + * + * @param {string} cell - CSV measurement cell value + * @param {CBQualitativeMeasurementTypeDefinition} measurement - Found qualitative measurement match + * @returns {*} + */ + _validateQualitativeMeasurementCell(cell: string, measurement: CBQualitativeMeasurementTypeDefinition) { + if (typeof cell !== 'string') { + return { error: 'Qualitative measurement expecting text value.', optionId: undefined }; + } + + const matchingOptionValue = measurement.options.find( + (option) => option.option_label.toLowerCase() === cell.toLowerCase() + ); + + // Validate cell value is an alowed qualitative measurement option + if (!matchingOptionValue) { + return { + error: `Incorrect qualitative measurement value. Allowed: ${measurement.options.map((option) => + option.option_label.toLowerCase() + )}`, + optionId: undefined + }; + } + + return { error: undefined, optionId: matchingOptionValue.qualitative_option_id }; + } + + /** + * Validate quantitative measurement + * + * @param {number} cell - CSV measurement cell value + * @param {CBQuantitativeMeasurementTypeDefinition} measurement - Found quantitative measurement match + * @returns {*} + */ + _validateQuantitativeMeasurementCell(cell: number, measurement: CBQuantitativeMeasurementTypeDefinition) { + if (typeof cell !== 'number') { + return { error: 'Quantitative measurement expecting number value.', value: undefined }; + } + + // Validate cell value is withing the measurement min max bounds + if (measurement.max_value != null && cell > measurement.max_value) { + return { error: 'Quantitative measurement out of bounds. Too small.', value: undefined }; + } + + if (measurement.min_value != null && cell < measurement.min_value) { + return { error: 'Quantitative measurement out of bounds. Too small.' }; + } + + return { error: undefined, value: cell }; + } + + /** + * Validate CSV worksheet rows against reference data. + * + * Note: This function is longer than I would like, but moving logic into seperate methods + * made the flow more complex and equally as long. + * + * @async + * @param {Row[]} rows - Invalidated CSV rows + * @param {WorkSheet} worksheet - Xlsx worksheet + * @returns {*} + */ + async validateRows(rows: Row[], worksheet: WorkSheet): Promise> { + // Generate type-safe cell getter from column validator + const nonStandardColumns = this._getNonStandardColumns(worksheet); + + // Get Critterbase reference data + const critterAliasMap = await this.surveyCritterService.getSurveyCritterAliasMap(this.surveyId); + const rowTsns = rows.map((row) => this._getRowMeta(row, critterAliasMap).tsn).filter(Boolean) as string[]; + const tsnMeasurementsMap = await this._getTsnsMeasurementMap(rowTsns); + + const rowErrors: ValidationError[] = []; + const validatedRows: CsvMeasurement[] = []; + + rows.forEach((row, index) => { + const { critter_id, capture_id, tsn } = this._getRowMeta(row, critterAliasMap); + + // Validate critter can be matched via alias + if (!critter_id) { + rowErrors.push({ row: index, message: 'Unable to find matching Critter with alias.' }); + return; + } + + // Validate capture can be matched with date and time + if (!capture_id) { + rowErrors.push({ row: index, message: 'Unable to find matching Capture with date and time.' }); + return; + } + + // This will only be triggered with an invalid alias + if (!tsn) { + rowErrors.push({ row: index, message: 'Unable to find ITIS TSN for Critter.' }); + return; + } + + // Loop through all non-standard (measurement) columns + for (const column of nonStandardColumns) { + // Get the cell value from the row (case insensitive) + const cellValue = row[column] ?? row[column.toLowerCase()] ?? row[column.toUpperCase()]; + + // If the cell value is null or undefined - skip validation + if (cellValue == null) { + continue; + } + + const measurements = tsnMeasurementsMap.get(tsn); + + // Validate taxon has reference measurements in Critterbase + if (!measurements || (!measurements.quantitative.length && !measurements.qualitative.length)) { + rowErrors.push({ row: index, col: column, message: 'No measurements exist for this taxon.' }); + continue; + } + + const qualitativeMeasurement = measurements?.qualitative.find( + (measurement) => measurement.measurement_name.toLowerCase() === column.toLowerCase() + ); + + // QUALITATIVE MEASUREMENT VALIDATION + if (qualitativeMeasurement) { + const { error, optionId } = this._validateQualitativeMeasurementCell(cellValue, qualitativeMeasurement); + + if (error !== undefined) { + rowErrors.push({ row: index, col: column, message: error }); + } else { + // Assign qualitative measurement to validated rows + validatedRows.push({ + critter_id, + capture_id, + taxon_measurement_id: qualitativeMeasurement.taxon_measurement_id, + qualitative_option_id: optionId + }); + } + + continue; + } + + const quantitativeMeasurement = measurements?.quantitative.find( + (measurement) => measurement.measurement_name.toLowerCase() === column.toLowerCase() + ); + + // QUANTITATIVE MEASUREMENT VALIDATION + if (quantitativeMeasurement) { + const { error, value } = this._validateQuantitativeMeasurementCell(cellValue, quantitativeMeasurement); + + if (error !== undefined) { + rowErrors.push({ row: index, col: column, message: error }); + } else { + // Assign quantitative measurement to validated rows + validatedRows.push({ + critter_id, + capture_id, + taxon_measurement_id: quantitativeMeasurement.taxon_measurement_id, + value: value + }); + } + + continue; + } + + // Validate the column header is a known Critterbase measurement + rowErrors.push({ + row: index, + col: column, + message: 'Unable to match column name to an existing measurement.' + }); + } + }); + + if (!rowErrors.length) { + return { success: true, data: validatedRows }; + } + + return { success: false, error: { issues: rowErrors } }; + } + + /** + * Insert CSV measurements into Critterbase. + * + * @async + * @param {CsvCritter[]} measurements - CSV row measurements + * @returns {Promise} List of inserted measurements + */ + async insert(measurements: CsvMeasurement[]): Promise { + const qualitative_measurements = measurements.filter( + (measurement): measurement is CsvQualitativeMeasurement => 'qualitative_option_id' in measurement + ); + + const quantitative_measurements = measurements.filter( + (measurement): measurement is CsvQuantitativeMeasurement => 'value' in measurement + ); + + const response = await this.surveyCritterService.critterbaseService.bulkCreate({ + qualitative_measurements, + quantitative_measurements + }); + + const measurementCount = response.created.qualitative_measurements + response.created.quantitative_measurements; + + defaultLog.debug({ label: 'import measurements', measurements, insertedCount: measurementCount }); + + return measurementCount; + } +} diff --git a/api/src/services/observation-service.ts b/api/src/services/observation-service.ts index b5ae70c09d..2d5d48f22d 100644 --- a/api/src/services/observation-service.ts +++ b/api/src/services/observation-service.ts @@ -43,7 +43,7 @@ import { validateMeasurements } from '../utils/observation-xlsx-utils/measurement-column-utils'; import { CSV_COLUMN_ALIASES } from '../utils/xlsx-utils/column-aliases'; -import { generateCellGetterFromColumnValidator } from '../utils/xlsx-utils/column-validator-utils'; +import { generateColumnCellGetterFromColumnValidator } from '../utils/xlsx-utils/column-validator-utils'; import { constructXLSXWorkbook, getDefaultWorksheet, @@ -82,7 +82,7 @@ export const observationStandardColumnValidator = { LONGITUDE: { type: 'number', aliases: CSV_COLUMN_ALIASES.LONGITUDE } } satisfies IXLSXCSVValidator; -export const getCellValue = generateCellGetterFromColumnValidator(observationStandardColumnValidator); +export const getColumnCellValue = generateColumnCellGetterFromColumnValidator(observationStandardColumnValidator); export interface InsertSubCount { observation_subcount_id: number | null; @@ -590,7 +590,7 @@ export class ObservationService extends DBService { const newRowData: InsertUpdateObservations[] = worksheetRowObjects.map((row) => { const newSubcount: InsertSubCount = { observation_subcount_id: null, - subcount: getCellValue(row, 'COUNT') as number, + subcount: getColumnCellValue(row, 'COUNT').cell as number, qualitative_measurements: [], quantitative_measurements: [], qualitative_environments: [], @@ -616,16 +616,16 @@ export class ObservationService extends DBService { return { standardColumns: { survey_id: surveyId, - itis_tsn: getCellValue(row, 'ITIS_TSN') as number, + itis_tsn: getColumnCellValue(row, 'ITIS_TSN').cell as number, itis_scientific_name: null, survey_sample_site_id: samplePeriodHierarchyIds?.survey_sample_site_id ?? null, survey_sample_method_id: samplePeriodHierarchyIds?.survey_sample_method_id ?? null, survey_sample_period_id: samplePeriodHierarchyIds?.survey_sample_period_id ?? null, - latitude: getCellValue(row, 'LATITUDE') as number, - longitude: getCellValue(row, 'LONGITUDE') as number, - count: getCellValue(row, 'COUNT') as number, - observation_time: getCellValue(row, 'TIME') as string, - observation_date: getCellValue(row, 'DATE') as string + latitude: getColumnCellValue(row, 'LATITUDE').cell as number, + longitude: getColumnCellValue(row, 'LONGITUDE').cell as number, + count: getColumnCellValue(row, 'COUNT').cell as number, + observation_time: getColumnCellValue(row, 'TIME').cell as string, + observation_date: getColumnCellValue(row, 'DATE').cell as string }, subcounts: [newSubcount] }; @@ -671,7 +671,7 @@ export class ObservationService extends DBService { } const measurement = getMeasurementFromTsnMeasurementTypeDefinitionMap( - getCellValue(row, 'ITIS_TSN') as string, + getColumnCellValue(row, 'ITIS_TSN').cell as string, mColumn, tsnMeasurements ); diff --git a/api/src/utils/xlsx-utils/column-validator-utils.test.ts b/api/src/utils/xlsx-utils/column-validator-utils.test.ts index 243c7fe8e3..46ff4044b0 100644 --- a/api/src/utils/xlsx-utils/column-validator-utils.test.ts +++ b/api/src/utils/xlsx-utils/column-validator-utils.test.ts @@ -1,6 +1,6 @@ import { expect } from 'chai'; import { - generateCellGetterFromColumnValidator, + generateColumnCellGetterFromColumnValidator, getColumnAliasesFromValidator, getColumnNamesFromValidator } from './column-validator-utils'; @@ -26,19 +26,27 @@ describe('column-validator-utils', () => { }); }); - describe('generateCellGetterFromColumnValidator', () => { - const getCellValue = generateCellGetterFromColumnValidator(columnValidator); + describe('generateColumnCellGetterFromColumnValidator', () => { + const getCellValue = generateColumnCellGetterFromColumnValidator(columnValidator); it('should return the cell value for a known column name', () => { - expect(getCellValue({ NAME: 'Dr. Steve Brule' }, 'NAME')).to.be.eql('Dr. Steve Brule'); + expect(getCellValue({ NAME: 'Dr. Steve Brule' }, 'NAME').cell).to.be.eql('Dr. Steve Brule'); }); it('should return the cell value for a known alias name', () => { - expect(getCellValue({ IDENTIFIER: 1 }, 'ID')).to.be.eql(1); + expect(getCellValue({ IDENTIFIER: 1 }, 'ID').cell).to.be.eql(1); }); it('should return undefined if row cannot find cell value', () => { - expect(getCellValue({ BAD_NAME: 1 }, 'NAME')).to.be.eql(undefined); + expect(getCellValue({ BAD_NAME: 1 }, 'NAME').cell).to.be.eql(undefined); + }); + + it('should return column name', () => { + expect(getCellValue({ NAME: 1 }, 'NAME').column).to.be.eql('NAME'); + }); + + it('should return column alias name', () => { + expect(getCellValue({ IDENTIFIER: 1 }, 'ID').column).to.be.eql('IDENTIFIER'); }); }); }); diff --git a/api/src/utils/xlsx-utils/column-validator-utils.ts b/api/src/utils/xlsx-utils/column-validator-utils.ts index b570cb321d..c1f4ff5384 100644 --- a/api/src/utils/xlsx-utils/column-validator-utils.ts +++ b/api/src/utils/xlsx-utils/column-validator-utils.ts @@ -51,29 +51,29 @@ export const getColumnValidatorSpecification = (columnValidator: IXLSXCSVValidat }; /** - * Generate a cell getter from a column validator. + * Generate a column + cell getter from a column validator. * - * Note: This will attempt to retrive the cell value from the row by the known header first. + * Note: This will attempt to retrive the column header and cell value from the row by the known header first. * If not found, it will then attempt to retrieve the value by the column header aliases. * - * TODO: Can the internal typing for this be improved (without the `as` cast)? - * * @example - * const getCellValue = generateCellGetterFromColumnValidator(columnValidator) - * const itis_tsn = getCellValue(row, 'ITIS_TSN') + * const getColumnCell = generateColumnCellGetterFromColumnValidator(columnValidator) + * + * const itis_tsn = getColumnCell(row, 'ITIS_TSN').cell + * const tsnColumn = getColumnCell(row, 'ITIS_TSN').column * * @template T * @param {T} columnValidator - Column validator * @returns {*} */ -export const generateCellGetterFromColumnValidator = (columnValidator: T) => { - return (row: Row, validatorKey: keyof T): J | undefined => { +export const generateColumnCellGetterFromColumnValidator = (columnValidator: T) => { + return (row: Row, validatorKey: keyof T): { column: string; cell: J | undefined } => { // Cast the columnValidatorKey to a string for convienience const key = validatorKey as string; - // Attempt to retrieve the cell value from the default column name + // Attempt to retrieve the column and cell value from the default column name if (row[key]) { - return row[key]; + return { column: key, cell: row[key] }; } const columnSpec = columnValidator[validatorKey] as IXLSXCSVColumn; @@ -81,11 +81,14 @@ export const generateCellGetterFromColumnValidator = { const worksheetRows = getWorksheetRowObjects(worksheet); const columnNames = getColumnNamesFromValidator(columnValidator); - const getCellValue = generateCellGetterFromColumnValidator(columnValidator); + const getCellValue = generateColumnCellGetterFromColumnValidator(columnValidator); return worksheetRows.every((row) => { return columnNames.every((columnName, index) => { - const value = getCellValue(row, columnName.toUpperCase() as Uppercase); + const value = getCellValue(row, columnName.toUpperCase() as Uppercase).cell; const type = typeof value; const columnSpec: IXLSXCSVColumn = columnValidator[columnName];