Skip to content

Commit

Permalink
#927
Browse files Browse the repository at this point in the history 
- changed how start and end dates are extracted for off-plot and on-plot protocols
- added dates to activity
- adds external id to activity
  • Loading branch information
@temi
temi committed May 2, 2024
1 parent 0a477d0 commit a718362
Show file tree
Hide file tree
Showing 6 changed files with 43 additions and 80 deletions.
2 changes: 1 addition & 1 deletion build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ plugins {
id "com.gorylenko.gradle-git-properties" version "2.4.1"
}

version "4.6-SNAPSHOT"
version "4.6-DATES-SNAPSHOT"
group "au.org.ala"
description "Ecodata"

Expand Down
21 changes: 13 additions & 8 deletions grails-app/services/au/org/ala/ecodata/ParatooService.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -297,23 +297,23 @@ class ParatooService {
surveyDataAndObservations[PARATOO_DATAMODEL_PLOT_LAYOUT] = dataSet.siteId
}

dataSet.startDate = config.getStartDate(surveyDataAndObservations)
dataSet.endDate = config.getEndDate(surveyDataAndObservations)
dataSet.format = DATASET_DATABASE_TABLE
dataSet.sizeUnknown = true

// Delete previously created activity so that duplicate species records are not created.
// Updating existing activity will also create duplicates since it relies on outputSpeciesId to determine
// if a record is new and new ones are created by code.
if (dataSet.activityId) {
activityService.delete(dataSet.activityId, true)
}

String activityId = createActivityFromSurveyData(form, surveyDataAndObservations, surveyId, dataSet.siteId, userId)
String activityId = createActivityFromSurveyData(form, surveyDataAndObservations, surveyId, dataSet, userId)
List records = recordService.getAllByActivity(activityId)
dataSet.areSpeciesRecorded = records?.size() > 0
dataSet.activityId = activityId

dataSet.startDate = config.getStartDate(surveyDataAndObservations)
dataSet.endDate = config.getEndDate(surveyDataAndObservations)
dataSet.format = DATASET_DATABASE_TABLE
dataSet.sizeUnknown = true

synchronized (LOCK) {
Map latestProject = projectService.get(project.project.projectId)
Map latestDataSet = latestProject.custom?.dataSets?.find { it.dataSetId == collection.orgMintedUUID }
Expand Down Expand Up @@ -456,14 +456,19 @@ class ParatooService {
* @param siteId
* @return
*/
private String createActivityFromSurveyData(ActivityForm activityForm, Map surveyObservations, ParatooCollectionId collection, String siteId, String userId) {
private String createActivityFromSurveyData(ActivityForm activityForm, Map surveyObservations, ParatooCollectionId collection, Map dataSet, String userId) {
Map activityProps = [
type : activityForm.name,
formVersion : activityForm.formVersion,
description : "Activity submitted by monitor",
projectId : collection.projectId,
publicationStatus: "published",
siteId : siteId,
siteId : dataSet.siteId,
startDate : dataSet.startDate,
endDate : dataSet.endDate,
plannedStartDate : dataSet.startDate,
plannedEndDate : dataSet.endDate,
externalIds : [new ExternalId(idType: ExternalId.IdType.MONITOR_MINTED_COLLECTION_ID, externalId: dataSet.dataSetId)],
userId : userId,
outputs : [[
data: surveyObservations,
Expand Down
65 changes: 10 additions & 55 deletions src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
package au.org.ala.ecodata.paratoo

import au.org.ala.ecodata.*
import au.org.ala.ecodata.converter.ISODateBindingConverter
import au.org.ala.ecodata.metadata.OutputMetadata
import au.org.ala.ecodata.metadata.PropertyAccessor
import com.fasterxml.jackson.annotation.JsonIgnoreProperties
Expand All @@ -25,9 +24,6 @@ class ParatooProtocolConfig {
String endDatePath = 'end_date_time'
String surveyIdPath = 'survey_metadata'
String plotVisitPath = 'plot_visit'
String plotProtocolObservationDatePath = "date_time"
String plotVisitStartDatePath = "${plotVisitPath}.start_date"
String plotVisitEndDatePath = "${plotVisitPath}.end_date"
String plotLayoutPath = "${plotVisitPath}.plot_layout"
String plotLayoutIdPath = "${plotLayoutPath}.id"
String plotLayoutPointsPath = "${plotLayoutPath}.plot_points"
Expand Down Expand Up @@ -55,46 +51,13 @@ class ParatooProtocolConfig {
return null
}

def date
if (usesPlotLayout) {
List dates = getDatesFromObservation(surveyData)
date = dates ? DateUtil.format(dates.first()) : null
return date
def date = getProperty(surveyData, startDatePath)
if (!date) {
date = getPropertyFromSurvey(surveyData, startDatePath)
}
else {
date = getProperty(surveyData, startDatePath)
if (!date) {
date = getPropertyFromSurvey(surveyData, startDatePath)
}

date = getFirst(date)
return removeMilliseconds(date)
}
}

/**
* Get date from plotProtocolObservationDatePath and sort them.
* @param surveyData - reverse lookup output which includes survey and observation data
* @return
*/
List getDatesFromObservation(Map surveyData) {
Map surveysData = surveyData.findAll { key, value ->
![ getSurveyAttributeName(), ParatooService.PARATOO_DATAMODEL_PLOT_SELECTION,
ParatooService.PARATOO_DATAMODEL_PLOT_VISIT, ParatooService.PARATOO_DATAMODEL_PLOT_LAYOUT].contains(key)
}
List result = []
ISODateBindingConverter converter = new ISODateBindingConverter()
surveysData.each { key, value ->
def dates = getProperty(value, plotProtocolObservationDatePath)
dates = dates instanceof List ? dates : [dates]

result.addAll(dates.collect { String date ->
date ? converter.convert(date, ISODateBindingConverter.FORMAT) : null
})
}

result = result.findAll { it != null }
result.sort()
date = getFirst(date)
return removeMilliseconds(date)
}

def getPropertyFromSurvey(Map surveyData, String path) {
Expand All @@ -107,21 +70,13 @@ class ParatooProtocolConfig {
return null
}

def date
if (usesPlotLayout) {
def dates = getDatesFromObservation(surveyData)
date = dates ? DateUtil.format(dates.last()) : null
return date
def date = getProperty(surveyData, endDatePath)
if (!date) {
date = getPropertyFromSurvey(surveyData, endDatePath)
}
else {
date = getProperty(surveyData, endDatePath)
if (!date) {
date = getPropertyFromSurvey(surveyData, endDatePath)
}

date = getFirst(date)
return removeMilliseconds(date)
}
date = getFirst(date)
return removeMilliseconds(date)
}

Map getSurveyId(Map surveyData) {
Expand Down
12 changes: 10 additions & 2 deletions src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -197,7 +197,11 @@ class ParatooServiceSpec extends MongoSpec implements ServiceUnitTest<ParatooSer
2 * projectService.get(projectId) >> [projectId: projectId, custom: [dataSets: [dataSet]]]
1 * projectService.update([custom: [dataSets: [expectedDataSetAsync]]], 'p1', false) >> [status: 'ok']
1 * projectService.update([custom: [dataSets: [expectedDataSetSync]]], 'p1', false) >> [status: 'ok']
1 * activityService.create(_) >> [activityId: '123']
1 * activityService.create({
it.startDate == "2023-09-01T00:00:00Z" && it.endDate == "2023-09-01T00:00:00Z" &&
it.plannedStartDate == "2023-09-01T00:00:00Z" && it.plannedEndDate == "2023-09-01T00:00:00Z" &&
it.externalIds[0].externalId == "d1" && it.externalIds[0].idType == ExternalId.IdType.MONITOR_MINTED_COLLECTION_ID
}) >> [activityId: '123']
1 * activityService.delete("123", true) >> [status: 'ok']
1 * recordService.getAllByActivity('123') >> []
1 * settingService.getSetting('paratoo.surveyData.mapping') >> {
Expand Down Expand Up @@ -307,7 +311,11 @@ class ParatooServiceSpec extends MongoSpec implements ServiceUnitTest<ParatooSer
2 * projectService.update(_, projectId, false) >> [status: 'ok']
2 * projectService.get(projectId) >> [projectId: projectId, custom: [dataSets: [dataSet]]]
1 * siteService.create(_) >> { site = it[0]; [siteId: 's1'] }
1 * activityService.create(_) >> [activityId: '123']
1 * activityService.create({
it.startDate == "2023-09-22T00:59:47Z" && it.endDate == "2023-09-23T00:59:47Z" &&
it.plannedStartDate == "2023-09-22T00:59:47Z" && it.plannedEndDate == "2023-09-23T00:59:47Z" &&
it.externalIds[0].externalId == "d1" && it.externalIds[0].idType == ExternalId.IdType.MONITOR_MINTED_COLLECTION_ID
}) >> [activityId: '123']
1 * recordService.getAllByActivity('123') >> []
1 * settingService.getSetting('paratoo.surveyData.mapping') >> {
(["guid-3": [
Expand Down
20 changes: 7 additions & 13 deletions src/test/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfigSpec.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -100,9 +100,7 @@ class ParatooProtocolConfigSpec extends Specification {
Map observation = apiOutput.collections
Map floristicsSurveyConfig = [
apiEndpoint:'floristics-veg-survey-lites',
usesPlotLayout:true,
startDatePath: 'plot_visit.start_date',
endDatePath: 'plot_visit.end_date'
usesPlotLayout:true
]
ParatooProtocolConfig config = new ParatooProtocolConfig(floristicsSurveyConfig)
config.setSurveyId(ParatooCollectionId.fromMap([survey_metadata: apiOutput.survey_metadata]))
Expand Down Expand Up @@ -134,8 +132,8 @@ class ParatooProtocolConfigSpec extends Specification {
transformData(observation, activityForm, config)

then:
config.getStartDate(observation) == "2024-04-08T01:23:28Z"
config.getEndDate(observation) == "2024-04-10T01:23:28Z"
config.getStartDate(observation) == "2022-09-21T01:55:44Z"
config.getEndDate(observation) == "2022-09-21T01:55:44Z"
config.getGeoJson(observation) == [type: "Feature", geometry: [type: "Polygon", coordinates: [[[152.880694, -27.388252], [152.880651, -27.388336], [152.880518, -27.388483], [152.880389, -27.388611], [152.88028, -27.388749], [152.880154, -27.388903], [152.880835, -27.389463], [152.880644, -27.389366], [152.880525, -27.389248], [152.88035, -27.389158], [152.880195, -27.389021], [152.880195, -27.389373], [152.880797, -27.388316], [152.881448, -27.388909], [152.881503, -27.388821], [152.881422, -27.388766], [152.881263, -27.388644], [152.881107, -27.388549], [152.880939, -27.388445], [152.881314, -27.389035], [152.88122, -27.389208], [152.881089, -27.389343], [152.880973, -27.389472], [152.880916, -27.389553], [152.880694, -27.388252]]]], properties: [name: "QDASEQ0001 - Control (100 x 100)", externalId: 1, description: "QDASEQ0001 - Control (100 x 100)", notes: "some comment"]]
}

Expand All @@ -144,9 +142,7 @@ class ParatooProtocolConfigSpec extends Specification {
Map surveyData = readSurveyData('basalAreaDbhReverseLookup')
Map basalAreaDbhMeasureSurveyConfig = [
apiEndpoint:'basal-area-dbh-measure-surveys',
usesPlotLayout:true,
startDatePath: 'start_date',
endDatePath: 'start_date',
usesPlotLayout:true
]
ParatooProtocolConfig config = new ParatooProtocolConfig(basalAreaDbhMeasureSurveyConfig)
config.setSurveyId(ParatooCollectionId.fromMap([survey_metadata: surveyData.survey_metadata]))
Expand Down Expand Up @@ -185,8 +181,8 @@ class ParatooProtocolConfigSpec extends Specification {
transformData(observation, activityForm, config)

expect:
config.getStartDate(observation) == "2024-03-28T03:17:01Z"
config.getEndDate(observation) == "2024-03-28T03:17:01Z"
config.getStartDate(observation) == "2023-09-22T00:59:47Z"
config.getEndDate(observation) == "2023-09-23T00:59:47Z"
config.getGeoJson(observation) == [
type : "Feature",
geometry : [
Expand All @@ -211,9 +207,7 @@ class ParatooProtocolConfigSpec extends Specification {
Map opportunisticSurveyConfig = [
apiEndpoint : 'opportunistic-surveys',
usesPlotLayout: false,
geometryType : 'Point',
startDatePath : 'start_date_time',
endDatePath : 'end_date_time'
geometryType : 'Point'
]
ActivityForm activityForm = new ActivityForm(
name: "aParatooForm 1",
Expand Down
3 changes: 2 additions & 1 deletion src/test/resources/paratoo/basalAreaDbhReverseLookup.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,8 @@
"system_org": "MERIT"
}
},
"start_date": "2023-09-22T00:59:47.807Z",
"start_date_time": "2023-09-22T00:59:47.807Z",
"end_date_time": "2023-09-23T00:59:47.807Z",
"createdAt": "2023-09-22T01:05:19.981Z",
"updatedAt": "2023-09-22T01:05:19.981Z",
"plot_size": {
Expand Down

0 comments on commit a718362

Please sign in to comment.