Merge branch 'dev' into feature/issue3171

build.gradle

@@ -20,7 +20,7 @@ plugins {
  id "com.gorylenko.gradle-git-properties" version "2.4.1"
 }
 
-version "4.6-SNAPSHOT"
+version "4.6-DATES-SNAPSHOT"
 group "au.org.ala"
 description "Ecodata"
 

grails-app/services/au/org/ala/ecodata/ParatooService.groovy

@@ -18,6 +18,7 @@ import static grails.async.Promises.task
  */
 @Slf4j
 class ParatooService {
+ static final Object LOCK = new Object()
  static final String DATASET_DATABASE_TABLE = 'Database Table'
  static final int PARATOO_MAX_RETRIES = 3
  static final String PARATOO_PROTOCOL_PATH = '/protocols'
@@ -182,18 +183,21 @@ class ParatooService {
 
  dataSet.surveyId = paratooCollectionId.toMap() // No codec to save this to mongo
 
- if (!project.custom) {
- project.custom = [:]
- }
- if (!project.custom.dataSets) {
- project.custom.dataSets = []
- }
-
  dataSet.orgMintedIdentifier = paratooCollectionId.encodeAsOrgMintedIdentifier()
 
  log.info "Minting identifier for Monitor collection: ${paratooCollectionId}: ${dataSet.orgMintedIdentifier}"
- project.custom.dataSets << dataSet
- Map result = projectService.update([custom: project.custom], projectId, false)
+ Map result
+ synchronized (LOCK) {
+ Map latestProject = projectService.get(projectId)
+ if (!latestProject.custom) {
+ latestProject.custom = [:]
+ }
+ if (!latestProject.custom.dataSets) {
+ latestProject.custom.dataSets = []
+ }
+ latestProject.custom.dataSets << dataSet
+ result = projectService.update([custom: latestProject.custom], projectId, false)
+ }
 
  if (!result.error) {
  result.orgMintedIdentifier = dataSet.orgMintedIdentifier
@@ -235,11 +239,19 @@ class ParatooService {
  log.error("An error occurred feching ${collection.orgMintedUUID}: ${e.message}", e)
  userService.clearCurrentUser()
  }
+
  promise.onComplete { Map result ->
  userService.clearCurrentUser()
  }
 
- def result = projectService.update([custom: project.project.custom], project.id, false)
+ def result
+ synchronized (LOCK) {
+ Map latestProject = projectService.get(project.id)
+ Map latestDataSet = latestProject.custom?.dataSets?.find { it.dataSetId == collection.orgMintedUUID }
+ latestDataSet.putAll(dataSet)
+ result = projectService.update([custom: latestProject.custom], project.id, false)
+ }
+
  [updateResult: result, promise: promise]
  }
 
@@ -291,24 +303,29 @@ class ParatooService {
  surveyDataAndObservations[PARATOO_DATAMODEL_PLOT_LAYOUT] = dataSet.siteId
  }
 
+ dataSet.startDate = config.getStartDate(surveyDataAndObservations)
+ dataSet.endDate = config.getEndDate(surveyDataAndObservations)
+ dataSet.format = DATASET_DATABASE_TABLE
+ dataSet.sizeUnknown = true
+
  // Delete previously created activity so that duplicate species records are not created.
  // Updating existing activity will also create duplicates since it relies on outputSpeciesId to determine
  // if a record is new and new ones are created by code.
  if (dataSet.activityId) {
  activityService.delete(dataSet.activityId, true)
  }
 
- String activityId = createActivityFromSurveyData(form, surveyDataAndObservations, surveyId, dataSet.siteId, userId)
+ String activityId = createActivityFromSurveyData(form, surveyDataAndObservations, surveyId, dataSet, userId)
  List records = recordService.getAllByActivity(activityId)
  dataSet.areSpeciesRecorded = records?.size() > 0
  dataSet.activityId = activityId
 
- dataSet.startDate = config.getStartDate(surveyDataAndObservations)
- dataSet.endDate = config.getEndDate(surveyDataAndObservations)
- dataSet.format = DATASET_DATABASE_TABLE
- dataSet.sizeUnknown = true
-
- projectService.update([custom: project.project.custom], project.id, false)
+ synchronized (LOCK) {
+  Map latestProject = projectService.get(project.project.projectId)
+  Map latestDataSet = latestProject.custom?.dataSets?.find { it.dataSetId == collection.orgMintedUUID }
+  latestDataSet.putAll(dataSet)
+ projectService.update([custom: latestProject.custom], project.id, false)
+ }
  }
  }
  }
@@ -445,14 +462,19 @@ class ParatooService {
  * @param siteId
  * @return
  */
- private String createActivityFromSurveyData(ActivityForm activityForm, Map surveyObservations, ParatooCollectionId collection, String siteId, String userId) {
+ private String createActivityFromSurveyData(ActivityForm activityForm, Map surveyObservations, ParatooCollectionId collection, Map dataSet, String userId) {
  Map activityProps = [
  type : activityForm.name,
  formVersion : activityForm.formVersion,
  description : "Activity submitted by monitor",
  projectId : collection.projectId,
  publicationStatus: "published",
- siteId : siteId,
+ siteId : dataSet.siteId,
+ startDate : dataSet.startDate,
+ endDate : dataSet.endDate,
+ plannedStartDate : dataSet.startDate,
+ plannedEndDate : dataSet.endDate,
+ externalIds : [new ExternalId(idType: ExternalId.IdType.MONITOR_MINTED_COLLECTION_ID, externalId: dataSet.dataSetId)],
  userId : userId,
  outputs : [[
  data: surveyObservations,

src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy

@@ -1,7 +1,6 @@
 package au.org.ala.ecodata.paratoo
 
 import au.org.ala.ecodata.*
-import au.org.ala.ecodata.converter.ISODateBindingConverter
 import au.org.ala.ecodata.metadata.OutputMetadata
 import au.org.ala.ecodata.metadata.PropertyAccessor
 import com.fasterxml.jackson.annotation.JsonIgnoreProperties
@@ -25,9 +24,6 @@ class ParatooProtocolConfig {
  String endDatePath = 'end_date_time'
  String surveyIdPath = 'survey_metadata'
  String plotVisitPath = 'plot_visit'
- String plotProtocolObservationDatePath = "date_time"
- String plotVisitStartDatePath = "${plotVisitPath}.start_date"
- String plotVisitEndDatePath = "${plotVisitPath}.end_date"
  String plotLayoutPath = "${plotVisitPath}.plot_layout"
  String plotLayoutIdPath = "${plotLayoutPath}.id"
  String plotLayoutPointsPath = "${plotLayoutPath}.plot_points"
@@ -55,46 +51,13 @@ class ParatooProtocolConfig {
  return null
  }
 
- def date
- if (usesPlotLayout) {
- List dates = getDatesFromObservation(surveyData)
- date = dates ? DateUtil.format(dates.first()) : null
- return date
+ def date = getProperty(surveyData, startDatePath)
+ if (!date) {
+ date = getPropertyFromSurvey(surveyData, startDatePath)
  }
- else {
- date = getProperty(surveyData, startDatePath)
- if (!date) {
- date = getPropertyFromSurvey(surveyData, startDatePath)
- }
 
- date = getFirst(date)
- return removeMilliseconds(date)
- }
- }
-
- /**
- * Get date from plotProtocolObservationDatePath and sort them.
- * @param surveyData - reverse lookup output which includes survey and observation data
- * @return
- */
- List getDatesFromObservation(Map surveyData) {
- Map surveysData = surveyData.findAll { key, value ->
- ![ getSurveyAttributeName(), ParatooService.PARATOO_DATAMODEL_PLOT_SELECTION,
- ParatooService.PARATOO_DATAMODEL_PLOT_VISIT, ParatooService.PARATOO_DATAMODEL_PLOT_LAYOUT].contains(key)
- }
- List result = []
- ISODateBindingConverter converter = new ISODateBindingConverter()
- surveysData.each { key, value ->
- def dates = getProperty(value, plotProtocolObservationDatePath)
- dates = dates instanceof List ? dates : [dates]
-
- result.addAll(dates.collect { String date ->
- date ? converter.convert(date, ISODateBindingConverter.FORMAT) : null
- })
- }
-
- result = result.findAll { it != null }
- result.sort()
+ date = getFirst(date)
+ return removeMilliseconds(date)
  }
 
  def getPropertyFromSurvey(Map surveyData, String path) {
@@ -107,21 +70,13 @@ class ParatooProtocolConfig {
  return null
  }
 
- def date
- if (usesPlotLayout) {
- def dates = getDatesFromObservation(surveyData)
- date = dates ? DateUtil.format(dates.last()) : null
- return date
+ def date = getProperty(surveyData, endDatePath)
+ if (!date) {
+ date = getPropertyFromSurvey(surveyData, endDatePath)
  }
- else {
- date = getProperty(surveyData, endDatePath)
- if (!date) {
- date = getPropertyFromSurvey(surveyData, endDatePath)
- }
 
- date = getFirst(date)
- return removeMilliseconds(date)
- }
+ date = getFirst(date)
+ return removeMilliseconds(date)
  }
 
  Map getSurveyId(Map surveyData) {

src/main/groovy/au/org/ala/ecodata/reporting/ReportGroups.groovy

@@ -119,6 +119,7 @@ class ReportGroups {
 
  static class DateGroup extends SinglePropertyGroupingStrategy {
 
+ static final String MISSING_DATE_GROUP_NAME = "Date missing"
  static DateTimeFormatter parser = ISODateTimeFormat.dateTimeNoMillis().withZone(DateTimeZone.default)
  DateTimeFormatter dateFormatter
  List buckets
@@ -157,6 +158,10 @@ class ReportGroups {
  def group(data) {
  def value = propertyAccessor.getPropertyValue(data)
 
+ if (!value) {
+ return MISSING_DATE_GROUP_NAME // Use a special group for null / empty dates.
+ }
+
  int result = bucketIndex(value)
 
  // we put results with an exact date match into the group where the end date of the bucket matches

src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy

@@ -142,11 +142,13 @@ class ParatooServiceSpec extends MongoSpec implements ServiceUnitTest<ParatooSer
  setup:
  ParatooCollectionId collectionId = buildCollectionId()
  String projectId = 'p1'
+ Map project = GormMongoUtil.extractDboProperties(getProject())
 
  when:
  Map result = service.mintCollectionId('u1', collectionId)
 
  then:
+ 1 * projectService.get(projectId) >> project
  1 * projectService.update(_, projectId, false) >> { data, pId, updateCollectory ->
  Map dataSet = data.custom.dataSets[1] // The stubbed project already has a dataSet, so the new one will be index=1
  assert dataSet.surveyId != null
@@ -192,9 +194,14 @@ class ParatooServiceSpec extends MongoSpec implements ServiceUnitTest<ParatooSer
  then:
  1 * webService.doPost(*_) >> [resp: [collections: ["coarse-woody-debris-survey": [uuid: "1", createdAt: "2023-09-01T00:00:00.123Z", start_date_time: "2023-09-01T00:00:00.123Z", end_date_time: "2023-09-01T00:00:00.123Z"]]]]
  1 * tokenService.getAuthToken(true) >> Mock(AccessToken)
+ 2 * projectService.get(projectId) >> [projectId: projectId, custom: [dataSets: [dataSet]]]
  1 * projectService.update([custom: [dataSets: [expectedDataSetAsync]]], 'p1', false) >> [status: 'ok']
  1 * projectService.update([custom: [dataSets: [expectedDataSetSync]]], 'p1', false) >> [status: 'ok']
- 1 * activityService.create(_) >> [activityId: '123']
+ 1 * activityService.create({
+ it.startDate == "2023-09-01T00:00:00Z" && it.endDate == "2023-09-01T00:00:00Z" &&
+ it.plannedStartDate == "2023-09-01T00:00:00Z" && it.plannedEndDate == "2023-09-01T00:00:00Z" &&
+ it.externalIds[0].externalId == "d1" && it.externalIds[0].idType == ExternalId.IdType.MONITOR_MINTED_COLLECTION_ID
+ }) >> [activityId: '123']
  1 * activityService.delete("123", true) >> [status: 'ok']
  1 * recordService.getAllByActivity('123') >> []
  1 * settingService.getSetting('paratoo.surveyData.mapping') >> {
@@ -304,8 +311,13 @@ class ParatooServiceSpec extends MongoSpec implements ServiceUnitTest<ParatooSer
  1 * webService.doPost(*_) >> [resp: surveyData]
  1 * tokenService.getAuthToken(true) >> Mock(AccessToken)
  2 * projectService.update(_, projectId, false) >> [status: 'ok']
+ 2 * projectService.get(projectId) >> [projectId: projectId, custom: [dataSets: [dataSet]]]
  1 * siteService.create(_) >> { site = it[0]; [siteId: 's1'] }
- 1 * activityService.create(_) >> [activityId: '123']
+ 1 * activityService.create({
+ it.startDate == "2023-09-22T00:59:47Z" && it.endDate == "2023-09-23T00:59:47Z" &&
+ it.plannedStartDate == "2023-09-22T00:59:47Z" && it.plannedEndDate == "2023-09-23T00:59:47Z" &&
+ it.externalIds[0].externalId == "d1" && it.externalIds[0].idType == ExternalId.IdType.MONITOR_MINTED_COLLECTION_ID
+ }) >> [activityId: '123']
  1 * recordService.getAllByActivity('123') >> []
  1 * settingService.getSetting('paratoo.surveyData.mapping') >> {
  (["guid-3": [
@@ -335,10 +347,8 @@ class ParatooServiceSpec extends MongoSpec implements ServiceUnitTest<ParatooSer
 
  }
 
- private void setupData() {
- Hub hub = new Hub(hubId: "merit", urlPath: "merit")
- hub.save(failOnError: true, flush: true)
- Project project = new Project(
+ private Map getProject(){
+ [
  projectId:"p1",
  name:"Project 1",
  grantId:"g1",
@@ -351,7 +361,14 @@ class ParatooServiceSpec extends MongoSpec implements ServiceUnitTest<ParatooSer
  monitoring:[rows:[[protocols:['protocol category 2', 'protocol category 3']]]]
  ], dataSets: [[
  dataSetId:'c1'
- ]]])
+ ]]]
+ ]
+ }
+
+ private void setupData() {
+ Hub hub = new Hub(hubId: "merit", urlPath: "merit")
+ hub.save(failOnError: true, flush: true)
+ Project project = new Project(getProject())
  project.save(failOnError: true, flush: true)
  UserPermission userPermission = new UserPermission(accessLevel: AccessLevel.admin, userId: userId, entityId: 'p1', entityType: Project.name)
  userPermission.save(failOnError: true, flush: true)

src/test/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfigSpec.groovy

@@ -100,9 +100,7 @@ class ParatooProtocolConfigSpec extends Specification {
  Map observation = apiOutput.collections
  Map floristicsSurveyConfig = [
  apiEndpoint:'floristics-veg-survey-lites',
- usesPlotLayout:true,
- startDatePath: 'plot_visit.start_date',
- endDatePath: 'plot_visit.end_date'
+ usesPlotLayout:true
  ]
  ParatooProtocolConfig config = new ParatooProtocolConfig(floristicsSurveyConfig)
  config.setSurveyId(ParatooCollectionId.fromMap([survey_metadata: apiOutput.survey_metadata]))
@@ -134,8 +132,8 @@ class ParatooProtocolConfigSpec extends Specification {
  transformData(observation, activityForm, config)
 
  then:
- config.getStartDate(observation) == "2024-04-08T01:23:28Z"
- config.getEndDate(observation) == "2024-04-10T01:23:28Z"
+ config.getStartDate(observation) == "2022-09-21T01:55:44Z"
+ config.getEndDate(observation) == "2022-09-21T01:55:44Z"
  config.getGeoJson(observation) == [type: "Feature", geometry: [type: "Polygon", coordinates: [[[152.880694, -27.388252], [152.880651, -27.388336], [152.880518, -27.388483], [152.880389, -27.388611], [152.88028, -27.388749], [152.880154, -27.388903], [152.880835, -27.389463], [152.880644, -27.389366], [152.880525, -27.389248], [152.88035, -27.389158], [152.880195, -27.389021], [152.880195, -27.389373], [152.880797, -27.388316], [152.881448, -27.388909], [152.881503, -27.388821], [152.881422, -27.388766], [152.881263, -27.388644], [152.881107, -27.388549], [152.880939, -27.388445], [152.881314, -27.389035], [152.88122, -27.389208], [152.881089, -27.389343], [152.880973, -27.389472], [152.880916, -27.389553], [152.880694, -27.388252]]]], properties: [name: "QDASEQ0001 - Control (100 x 100)", externalId: 1, description: "QDASEQ0001 - Control (100 x 100)", notes: "some comment"]]
  }
 
@@ -144,9 +142,7 @@ class ParatooProtocolConfigSpec extends Specification {
  Map surveyData = readSurveyData('basalAreaDbhReverseLookup')
  Map basalAreaDbhMeasureSurveyConfig = [
  apiEndpoint:'basal-area-dbh-measure-surveys',
- usesPlotLayout:true,
- startDatePath: 'start_date',
- endDatePath: 'start_date',
+ usesPlotLayout:true
  ]
  ParatooProtocolConfig config = new ParatooProtocolConfig(basalAreaDbhMeasureSurveyConfig)
  config.setSurveyId(ParatooCollectionId.fromMap([survey_metadata: surveyData.survey_metadata]))
@@ -185,8 +181,8 @@ class ParatooProtocolConfigSpec extends Specification {
  transformData(observation, activityForm, config)
 
  expect:
- config.getStartDate(observation) == "2024-03-28T03:17:01Z"
- config.getEndDate(observation) == "2024-03-28T03:17:01Z"
+ config.getStartDate(observation) == "2023-09-22T00:59:47Z"
+ config.getEndDate(observation) == "2023-09-23T00:59:47Z"
  config.getGeoJson(observation) == [
  type : "Feature",
  geometry : [
@@ -211,9 +207,7 @@ class ParatooProtocolConfigSpec extends Specification {
  Map opportunisticSurveyConfig = [
  apiEndpoint : 'opportunistic-surveys',
  usesPlotLayout: false,
- geometryType : 'Point',
- startDatePath : 'start_date_time',
- endDatePath : 'end_date_time'
+ geometryType : 'Point'
  ]
  ActivityForm activityForm = new ActivityForm(
  name: "aParatooForm 1",

src/test/groovy/au/org/ala/ecodata/reporting/ReportGroupsSpec.groovy

@@ -105,6 +105,8 @@ public class ReportGroupsSpec extends Specification {
  ['2014-09-01T00:00:00Z', '2014-10-01T00:00:00Z', '2014-11-01T00:00:00Z'] | '2014-08-13T00:00:00Z' | 'MM-yyyy' | 'Before 09-2014'
  ['2014-09-01T00:00:00Z', '2014-10-01T00:00:00Z', '2014-11-01T00:00:00Z'] | '2014-11-01T00:00:01Z' | 'MM-yyyy' | 'After 10-2014'
  ['2014-10-01T00:00:00Z', '2015-01-01T00:00:00Z', '2015-04-01T00:00:00Z'] | '2014-10-13T00:00:00Z' | 'MM-yyyy' | '10-2014 - 12-2014'
+ ['2014-10-01T00:00:00Z', '2015-01-01T00:00:00Z', '2015-04-01T00:00:00Z'] | null | 'MM-yyyy' | 'Date missing'
+ ['2014-10-01T00:00:00Z', '2015-01-01T00:00:00Z', '2015-04-01T00:00:00Z'] | '' | 'MM-yyyy' | 'Date missing'
 
 
  }

src/test/resources/paratoo/basalAreaDbhReverseLookup.json

@@ -41,7 +41,8 @@
  "system_org": "MERIT"
  }
  },
- "start_date": "2023-09-22T00:59:47.807Z",
+ "start_date_time": "2023-09-22T00:59:47.807Z",
+ "end_date_time": "2023-09-23T00:59:47.807Z",
  "createdAt": "2023-09-22T01:05:19.981Z",
  "updatedAt": "2023-09-22T01:05:19.981Z",
  "plot_size": {