diff --git a/build.gradle b/build.gradle index 56e413bff..21e18e5cc 100644 --- a/build.gradle +++ b/build.gradle @@ -20,7 +20,7 @@ plugins { id "com.gorylenko.gradle-git-properties" version "2.4.1" } -version "4.6-SNAPSHOT" +version "4.6-DATES-SNAPSHOT" group "au.org.ala" description "Ecodata" diff --git a/grails-app/services/au/org/ala/ecodata/ParatooService.groovy b/grails-app/services/au/org/ala/ecodata/ParatooService.groovy index 7e2ee8013..86044d2dd 100644 --- a/grails-app/services/au/org/ala/ecodata/ParatooService.groovy +++ b/grails-app/services/au/org/ala/ecodata/ParatooService.groovy @@ -297,6 +297,11 @@ class ParatooService { surveyDataAndObservations[PARATOO_DATAMODEL_PLOT_LAYOUT] = dataSet.siteId } + dataSet.startDate = config.getStartDate(surveyDataAndObservations) + dataSet.endDate = config.getEndDate(surveyDataAndObservations) + dataSet.format = DATASET_DATABASE_TABLE + dataSet.sizeUnknown = true + // Delete previously created activity so that duplicate species records are not created. // Updating existing activity will also create duplicates since it relies on outputSpeciesId to determine // if a record is new and new ones are created by code. @@ -304,16 +309,11 @@ class ParatooService { activityService.delete(dataSet.activityId, true) } - String activityId = createActivityFromSurveyData(form, surveyDataAndObservations, surveyId, dataSet.siteId, userId) + String activityId = createActivityFromSurveyData(form, surveyDataAndObservations, surveyId, dataSet, userId) List records = recordService.getAllByActivity(activityId) dataSet.areSpeciesRecorded = records?.size() > 0 dataSet.activityId = activityId - dataSet.startDate = config.getStartDate(surveyDataAndObservations) - dataSet.endDate = config.getEndDate(surveyDataAndObservations) - dataSet.format = DATASET_DATABASE_TABLE - dataSet.sizeUnknown = true - synchronized (LOCK) { Map latestProject = projectService.get(project.project.projectId) Map latestDataSet = latestProject.custom?.dataSets?.find { it.dataSetId == collection.orgMintedUUID } @@ -456,14 +456,19 @@ class ParatooService { * @param siteId * @return */ - private String createActivityFromSurveyData(ActivityForm activityForm, Map surveyObservations, ParatooCollectionId collection, String siteId, String userId) { + private String createActivityFromSurveyData(ActivityForm activityForm, Map surveyObservations, ParatooCollectionId collection, Map dataSet, String userId) { Map activityProps = [ type : activityForm.name, formVersion : activityForm.formVersion, description : "Activity submitted by monitor", projectId : collection.projectId, publicationStatus: "published", - siteId : siteId, + siteId : dataSet.siteId, + startDate : dataSet.startDate, + endDate : dataSet.endDate, + plannedStartDate : dataSet.startDate, + plannedEndDate : dataSet.endDate, + externalIds : [new ExternalId(idType: ExternalId.IdType.MONITOR_MINTED_COLLECTION_ID, externalId: dataSet.dataSetId)], userId : userId, outputs : [[ data: surveyObservations, diff --git a/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy b/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy index e6b0be669..9ccd795c1 100644 --- a/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy +++ b/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy @@ -1,7 +1,6 @@ package au.org.ala.ecodata.paratoo import au.org.ala.ecodata.* -import au.org.ala.ecodata.converter.ISODateBindingConverter import au.org.ala.ecodata.metadata.OutputMetadata import au.org.ala.ecodata.metadata.PropertyAccessor import com.fasterxml.jackson.annotation.JsonIgnoreProperties @@ -25,9 +24,6 @@ class ParatooProtocolConfig { String endDatePath = 'end_date_time' String surveyIdPath = 'survey_metadata' String plotVisitPath = 'plot_visit' - String plotProtocolObservationDatePath = "date_time" - String plotVisitStartDatePath = "${plotVisitPath}.start_date" - String plotVisitEndDatePath = "${plotVisitPath}.end_date" String plotLayoutPath = "${plotVisitPath}.plot_layout" String plotLayoutIdPath = "${plotLayoutPath}.id" String plotLayoutPointsPath = "${plotLayoutPath}.plot_points" @@ -55,46 +51,13 @@ class ParatooProtocolConfig { return null } - def date - if (usesPlotLayout) { - List dates = getDatesFromObservation(surveyData) - date = dates ? DateUtil.format(dates.first()) : null - return date + def date = getProperty(surveyData, startDatePath) + if (!date) { + date = getPropertyFromSurvey(surveyData, startDatePath) } - else { - date = getProperty(surveyData, startDatePath) - if (!date) { - date = getPropertyFromSurvey(surveyData, startDatePath) - } - date = getFirst(date) - return removeMilliseconds(date) - } - } - - /** - * Get date from plotProtocolObservationDatePath and sort them. - * @param surveyData - reverse lookup output which includes survey and observation data - * @return - */ - List getDatesFromObservation(Map surveyData) { - Map surveysData = surveyData.findAll { key, value -> - ![ getSurveyAttributeName(), ParatooService.PARATOO_DATAMODEL_PLOT_SELECTION, - ParatooService.PARATOO_DATAMODEL_PLOT_VISIT, ParatooService.PARATOO_DATAMODEL_PLOT_LAYOUT].contains(key) - } - List result = [] - ISODateBindingConverter converter = new ISODateBindingConverter() - surveysData.each { key, value -> - def dates = getProperty(value, plotProtocolObservationDatePath) - dates = dates instanceof List ? dates : [dates] - - result.addAll(dates.collect { String date -> - date ? converter.convert(date, ISODateBindingConverter.FORMAT) : null - }) - } - - result = result.findAll { it != null } - result.sort() + date = getFirst(date) + return removeMilliseconds(date) } def getPropertyFromSurvey(Map surveyData, String path) { @@ -107,21 +70,13 @@ class ParatooProtocolConfig { return null } - def date - if (usesPlotLayout) { - def dates = getDatesFromObservation(surveyData) - date = dates ? DateUtil.format(dates.last()) : null - return date + def date = getProperty(surveyData, endDatePath) + if (!date) { + date = getPropertyFromSurvey(surveyData, endDatePath) } - else { - date = getProperty(surveyData, endDatePath) - if (!date) { - date = getPropertyFromSurvey(surveyData, endDatePath) - } - date = getFirst(date) - return removeMilliseconds(date) - } + date = getFirst(date) + return removeMilliseconds(date) } Map getSurveyId(Map surveyData) { diff --git a/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy b/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy index f79449bdf..5cf4b473b 100644 --- a/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy +++ b/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy @@ -197,7 +197,11 @@ class ParatooServiceSpec extends MongoSpec implements ServiceUnitTest> [projectId: projectId, custom: [dataSets: [dataSet]]] 1 * projectService.update([custom: [dataSets: [expectedDataSetAsync]]], 'p1', false) >> [status: 'ok'] 1 * projectService.update([custom: [dataSets: [expectedDataSetSync]]], 'p1', false) >> [status: 'ok'] - 1 * activityService.create(_) >> [activityId: '123'] + 1 * activityService.create({ + it.startDate == "2023-09-01T00:00:00Z" && it.endDate == "2023-09-01T00:00:00Z" && + it.plannedStartDate == "2023-09-01T00:00:00Z" && it.plannedEndDate == "2023-09-01T00:00:00Z" && + it.externalIds[0].externalId == "d1" && it.externalIds[0].idType == ExternalId.IdType.MONITOR_MINTED_COLLECTION_ID + }) >> [activityId: '123'] 1 * activityService.delete("123", true) >> [status: 'ok'] 1 * recordService.getAllByActivity('123') >> [] 1 * settingService.getSetting('paratoo.surveyData.mapping') >> { @@ -307,7 +311,11 @@ class ParatooServiceSpec extends MongoSpec implements ServiceUnitTest> [status: 'ok'] 2 * projectService.get(projectId) >> [projectId: projectId, custom: [dataSets: [dataSet]]] 1 * siteService.create(_) >> { site = it[0]; [siteId: 's1'] } - 1 * activityService.create(_) >> [activityId: '123'] + 1 * activityService.create({ + it.startDate == "2023-09-22T00:59:47Z" && it.endDate == "2023-09-23T00:59:47Z" && + it.plannedStartDate == "2023-09-22T00:59:47Z" && it.plannedEndDate == "2023-09-23T00:59:47Z" && + it.externalIds[0].externalId == "d1" && it.externalIds[0].idType == ExternalId.IdType.MONITOR_MINTED_COLLECTION_ID + }) >> [activityId: '123'] 1 * recordService.getAllByActivity('123') >> [] 1 * settingService.getSetting('paratoo.surveyData.mapping') >> { (["guid-3": [ diff --git a/src/test/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfigSpec.groovy b/src/test/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfigSpec.groovy index dbfbf87fa..9b2560aac 100644 --- a/src/test/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfigSpec.groovy +++ b/src/test/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfigSpec.groovy @@ -100,9 +100,7 @@ class ParatooProtocolConfigSpec extends Specification { Map observation = apiOutput.collections Map floristicsSurveyConfig = [ apiEndpoint:'floristics-veg-survey-lites', - usesPlotLayout:true, - startDatePath: 'plot_visit.start_date', - endDatePath: 'plot_visit.end_date' + usesPlotLayout:true ] ParatooProtocolConfig config = new ParatooProtocolConfig(floristicsSurveyConfig) config.setSurveyId(ParatooCollectionId.fromMap([survey_metadata: apiOutput.survey_metadata])) @@ -134,8 +132,8 @@ class ParatooProtocolConfigSpec extends Specification { transformData(observation, activityForm, config) then: - config.getStartDate(observation) == "2024-04-08T01:23:28Z" - config.getEndDate(observation) == "2024-04-10T01:23:28Z" + config.getStartDate(observation) == "2022-09-21T01:55:44Z" + config.getEndDate(observation) == "2022-09-21T01:55:44Z" config.getGeoJson(observation) == [type: "Feature", geometry: [type: "Polygon", coordinates: [[[152.880694, -27.388252], [152.880651, -27.388336], [152.880518, -27.388483], [152.880389, -27.388611], [152.88028, -27.388749], [152.880154, -27.388903], [152.880835, -27.389463], [152.880644, -27.389366], [152.880525, -27.389248], [152.88035, -27.389158], [152.880195, -27.389021], [152.880195, -27.389373], [152.880797, -27.388316], [152.881448, -27.388909], [152.881503, -27.388821], [152.881422, -27.388766], [152.881263, -27.388644], [152.881107, -27.388549], [152.880939, -27.388445], [152.881314, -27.389035], [152.88122, -27.389208], [152.881089, -27.389343], [152.880973, -27.389472], [152.880916, -27.389553], [152.880694, -27.388252]]]], properties: [name: "QDASEQ0001 - Control (100 x 100)", externalId: 1, description: "QDASEQ0001 - Control (100 x 100)", notes: "some comment"]] } @@ -144,9 +142,7 @@ class ParatooProtocolConfigSpec extends Specification { Map surveyData = readSurveyData('basalAreaDbhReverseLookup') Map basalAreaDbhMeasureSurveyConfig = [ apiEndpoint:'basal-area-dbh-measure-surveys', - usesPlotLayout:true, - startDatePath: 'start_date', - endDatePath: 'start_date', + usesPlotLayout:true ] ParatooProtocolConfig config = new ParatooProtocolConfig(basalAreaDbhMeasureSurveyConfig) config.setSurveyId(ParatooCollectionId.fromMap([survey_metadata: surveyData.survey_metadata])) @@ -185,8 +181,8 @@ class ParatooProtocolConfigSpec extends Specification { transformData(observation, activityForm, config) expect: - config.getStartDate(observation) == "2024-03-28T03:17:01Z" - config.getEndDate(observation) == "2024-03-28T03:17:01Z" + config.getStartDate(observation) == "2023-09-22T00:59:47Z" + config.getEndDate(observation) == "2023-09-23T00:59:47Z" config.getGeoJson(observation) == [ type : "Feature", geometry : [ @@ -211,9 +207,7 @@ class ParatooProtocolConfigSpec extends Specification { Map opportunisticSurveyConfig = [ apiEndpoint : 'opportunistic-surveys', usesPlotLayout: false, - geometryType : 'Point', - startDatePath : 'start_date_time', - endDatePath : 'end_date_time' + geometryType : 'Point' ] ActivityForm activityForm = new ActivityForm( name: "aParatooForm 1", diff --git a/src/test/resources/paratoo/basalAreaDbhReverseLookup.json b/src/test/resources/paratoo/basalAreaDbhReverseLookup.json index 27145c2ab..15ab3ed80 100644 --- a/src/test/resources/paratoo/basalAreaDbhReverseLookup.json +++ b/src/test/resources/paratoo/basalAreaDbhReverseLookup.json @@ -41,7 +41,8 @@ "system_org": "MERIT" } }, - "start_date": "2023-09-22T00:59:47.807Z", + "start_date_time": "2023-09-22T00:59:47.807Z", + "end_date_time": "2023-09-23T00:59:47.807Z", "createdAt": "2023-09-22T01:05:19.981Z", "updatedAt": "2023-09-22T01:05:19.981Z", "plot_size": {