From 6170591e6883c6ca30ce63311780491ef16286ea Mon Sep 17 00:00:00 2001 From: chrisala Date: Mon, 29 Apr 2024 11:11:58 +1000 Subject: [PATCH 01/44] 4.6-SNAPSHOT --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 93e530559..56e413bff 100644 --- a/build.gradle +++ b/build.gradle @@ -20,7 +20,7 @@ plugins { id "com.gorylenko.gradle-git-properties" version "2.4.1" } -version "4.5" +version "4.6-SNAPSHOT" group "au.org.ala" description "Ecodata" From 9a0ebdc22938865a82ad8c37f0f592406444a260 Mon Sep 17 00:00:00 2001 From: chrisala Date: Wed, 1 May 2024 10:01:40 +1000 Subject: [PATCH 02/44] Handle grouping null dates in DateGroup #928 --- .../groovy/au/org/ala/ecodata/reporting/ReportGroups.groovy | 5 +++++ .../au/org/ala/ecodata/reporting/ReportGroupsSpec.groovy | 2 ++ 2 files changed, 7 insertions(+) diff --git a/src/main/groovy/au/org/ala/ecodata/reporting/ReportGroups.groovy b/src/main/groovy/au/org/ala/ecodata/reporting/ReportGroups.groovy index bbeeb7be5..d8f3ba5fd 100644 --- a/src/main/groovy/au/org/ala/ecodata/reporting/ReportGroups.groovy +++ b/src/main/groovy/au/org/ala/ecodata/reporting/ReportGroups.groovy @@ -119,6 +119,7 @@ class ReportGroups { static class DateGroup extends SinglePropertyGroupingStrategy { + static final String MISSING_DATE_GROUP_NAME = "Date missing" static DateTimeFormatter parser = ISODateTimeFormat.dateTimeNoMillis().withZone(DateTimeZone.default) DateTimeFormatter dateFormatter List buckets @@ -157,6 +158,10 @@ class ReportGroups { def group(data) { def value = propertyAccessor.getPropertyValue(data) + if (!value) { + return MISSING_DATE_GROUP_NAME // Use a special group for null / empty dates. + } + int result = bucketIndex(value) // we put results with an exact date match into the group where the end date of the bucket matches diff --git a/src/test/groovy/au/org/ala/ecodata/reporting/ReportGroupsSpec.groovy b/src/test/groovy/au/org/ala/ecodata/reporting/ReportGroupsSpec.groovy index 81679f6fd..11408dbda 100644 --- a/src/test/groovy/au/org/ala/ecodata/reporting/ReportGroupsSpec.groovy +++ b/src/test/groovy/au/org/ala/ecodata/reporting/ReportGroupsSpec.groovy @@ -105,6 +105,8 @@ public class ReportGroupsSpec extends Specification { ['2014-09-01T00:00:00Z', '2014-10-01T00:00:00Z', '2014-11-01T00:00:00Z'] | '2014-08-13T00:00:00Z' | 'MM-yyyy' | 'Before 09-2014' ['2014-09-01T00:00:00Z', '2014-10-01T00:00:00Z', '2014-11-01T00:00:00Z'] | '2014-11-01T00:00:01Z' | 'MM-yyyy' | 'After 10-2014' ['2014-10-01T00:00:00Z', '2015-01-01T00:00:00Z', '2015-04-01T00:00:00Z'] | '2014-10-13T00:00:00Z' | 'MM-yyyy' | '10-2014 - 12-2014' + ['2014-10-01T00:00:00Z', '2015-01-01T00:00:00Z', '2015-04-01T00:00:00Z'] | null | 'MM-yyyy' | 'Date missing' + ['2014-10-01T00:00:00Z', '2015-01-01T00:00:00Z', '2015-04-01T00:00:00Z'] | '' | 'MM-yyyy' | 'Date missing' } From 5899f31da01f9dbca95bb0eeb1a799a762399d10 Mon Sep 17 00:00:00 2001 From: temi Date: Wed, 1 May 2024 14:25:43 +1000 Subject: [PATCH 03/44] AtlasOfLivingAustralia/fieldcapture#3171 - track user in audit messages --- .../services/au/org/ala/ecodata/ParatooService.groovy | 6 ++++++ .../groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy | 3 +++ 2 files changed, 9 insertions(+) diff --git a/grails-app/services/au/org/ala/ecodata/ParatooService.groovy b/grails-app/services/au/org/ala/ecodata/ParatooService.groovy index c0d745bf9..75942f019 100644 --- a/grails-app/services/au/org/ala/ecodata/ParatooService.groovy +++ b/grails-app/services/au/org/ala/ecodata/ParatooService.groovy @@ -228,11 +228,17 @@ class ParatooService { Map authHeader = getAuthHeader() Promise promise = task { + userService.setCurrentUser(userId) asyncFetchCollection(collection, authHeader, userId, project) } promise.onError { Throwable e -> log.error("An error occurred feching ${collection.orgMintedUUID}: ${e.message}", e) + userService.clearCurrentUser() } + promise.onComplete { Map result -> + userService.clearCurrentUser() + } + def result = projectService.update([custom: project.project.custom], project.id, false) [updateResult: result, promise: promise] } diff --git a/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy b/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy index a57dc2dfe..3bb8f0642 100644 --- a/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy +++ b/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy @@ -210,6 +210,7 @@ class ParatooServiceSpec extends MongoSpec implements ServiceUnitTest> [userId: userId] + 1 * userService.setCurrentUser(userId) and: result.updateResult == [status: 'ok'] @@ -297,6 +298,7 @@ class ParatooServiceSpec extends MongoSpec implements ServiceUnitTest> [resp: surveyData] @@ -318,6 +320,7 @@ class ParatooServiceSpec extends MongoSpec implements ServiceUnitTest> [userId: userId] + 1 * userService.setCurrentUser(userId) and: site.name == "SATFLB0001 - Control (100 x 100)" From a718362eec2bc833f64ddade98527c47934331f7 Mon Sep 17 00:00:00 2001 From: temi Date: Thu, 2 May 2024 11:39:42 +1000 Subject: [PATCH 04/44] #927 - changed how start and end dates are extracted for off-plot and on-plot protocols - added dates to activity - adds external id to activity --- build.gradle | 2 +- .../au/org/ala/ecodata/ParatooService.groovy | 21 +++--- .../paratoo/ParatooProtocolConfig.groovy | 65 +++---------------- .../org/ala/ecodata/ParatooServiceSpec.groovy | 12 +++- .../paratoo/ParatooProtocolConfigSpec.groovy | 20 ++---- .../paratoo/basalAreaDbhReverseLookup.json | 3 +- 6 files changed, 43 insertions(+), 80 deletions(-) diff --git a/build.gradle b/build.gradle index 56e413bff..21e18e5cc 100644 --- a/build.gradle +++ b/build.gradle @@ -20,7 +20,7 @@ plugins { id "com.gorylenko.gradle-git-properties" version "2.4.1" } -version "4.6-SNAPSHOT" +version "4.6-DATES-SNAPSHOT" group "au.org.ala" description "Ecodata" diff --git a/grails-app/services/au/org/ala/ecodata/ParatooService.groovy b/grails-app/services/au/org/ala/ecodata/ParatooService.groovy index 7e2ee8013..86044d2dd 100644 --- a/grails-app/services/au/org/ala/ecodata/ParatooService.groovy +++ b/grails-app/services/au/org/ala/ecodata/ParatooService.groovy @@ -297,6 +297,11 @@ class ParatooService { surveyDataAndObservations[PARATOO_DATAMODEL_PLOT_LAYOUT] = dataSet.siteId } + dataSet.startDate = config.getStartDate(surveyDataAndObservations) + dataSet.endDate = config.getEndDate(surveyDataAndObservations) + dataSet.format = DATASET_DATABASE_TABLE + dataSet.sizeUnknown = true + // Delete previously created activity so that duplicate species records are not created. // Updating existing activity will also create duplicates since it relies on outputSpeciesId to determine // if a record is new and new ones are created by code. @@ -304,16 +309,11 @@ class ParatooService { activityService.delete(dataSet.activityId, true) } - String activityId = createActivityFromSurveyData(form, surveyDataAndObservations, surveyId, dataSet.siteId, userId) + String activityId = createActivityFromSurveyData(form, surveyDataAndObservations, surveyId, dataSet, userId) List records = recordService.getAllByActivity(activityId) dataSet.areSpeciesRecorded = records?.size() > 0 dataSet.activityId = activityId - dataSet.startDate = config.getStartDate(surveyDataAndObservations) - dataSet.endDate = config.getEndDate(surveyDataAndObservations) - dataSet.format = DATASET_DATABASE_TABLE - dataSet.sizeUnknown = true - synchronized (LOCK) { Map latestProject = projectService.get(project.project.projectId) Map latestDataSet = latestProject.custom?.dataSets?.find { it.dataSetId == collection.orgMintedUUID } @@ -456,14 +456,19 @@ class ParatooService { * @param siteId * @return */ - private String createActivityFromSurveyData(ActivityForm activityForm, Map surveyObservations, ParatooCollectionId collection, String siteId, String userId) { + private String createActivityFromSurveyData(ActivityForm activityForm, Map surveyObservations, ParatooCollectionId collection, Map dataSet, String userId) { Map activityProps = [ type : activityForm.name, formVersion : activityForm.formVersion, description : "Activity submitted by monitor", projectId : collection.projectId, publicationStatus: "published", - siteId : siteId, + siteId : dataSet.siteId, + startDate : dataSet.startDate, + endDate : dataSet.endDate, + plannedStartDate : dataSet.startDate, + plannedEndDate : dataSet.endDate, + externalIds : [new ExternalId(idType: ExternalId.IdType.MONITOR_MINTED_COLLECTION_ID, externalId: dataSet.dataSetId)], userId : userId, outputs : [[ data: surveyObservations, diff --git a/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy b/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy index e6b0be669..9ccd795c1 100644 --- a/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy +++ b/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy @@ -1,7 +1,6 @@ package au.org.ala.ecodata.paratoo import au.org.ala.ecodata.* -import au.org.ala.ecodata.converter.ISODateBindingConverter import au.org.ala.ecodata.metadata.OutputMetadata import au.org.ala.ecodata.metadata.PropertyAccessor import com.fasterxml.jackson.annotation.JsonIgnoreProperties @@ -25,9 +24,6 @@ class ParatooProtocolConfig { String endDatePath = 'end_date_time' String surveyIdPath = 'survey_metadata' String plotVisitPath = 'plot_visit' - String plotProtocolObservationDatePath = "date_time" - String plotVisitStartDatePath = "${plotVisitPath}.start_date" - String plotVisitEndDatePath = "${plotVisitPath}.end_date" String plotLayoutPath = "${plotVisitPath}.plot_layout" String plotLayoutIdPath = "${plotLayoutPath}.id" String plotLayoutPointsPath = "${plotLayoutPath}.plot_points" @@ -55,46 +51,13 @@ class ParatooProtocolConfig { return null } - def date - if (usesPlotLayout) { - List dates = getDatesFromObservation(surveyData) - date = dates ? DateUtil.format(dates.first()) : null - return date + def date = getProperty(surveyData, startDatePath) + if (!date) { + date = getPropertyFromSurvey(surveyData, startDatePath) } - else { - date = getProperty(surveyData, startDatePath) - if (!date) { - date = getPropertyFromSurvey(surveyData, startDatePath) - } - date = getFirst(date) - return removeMilliseconds(date) - } - } - - /** - * Get date from plotProtocolObservationDatePath and sort them. - * @param surveyData - reverse lookup output which includes survey and observation data - * @return - */ - List getDatesFromObservation(Map surveyData) { - Map surveysData = surveyData.findAll { key, value -> - ![ getSurveyAttributeName(), ParatooService.PARATOO_DATAMODEL_PLOT_SELECTION, - ParatooService.PARATOO_DATAMODEL_PLOT_VISIT, ParatooService.PARATOO_DATAMODEL_PLOT_LAYOUT].contains(key) - } - List result = [] - ISODateBindingConverter converter = new ISODateBindingConverter() - surveysData.each { key, value -> - def dates = getProperty(value, plotProtocolObservationDatePath) - dates = dates instanceof List ? dates : [dates] - - result.addAll(dates.collect { String date -> - date ? converter.convert(date, ISODateBindingConverter.FORMAT) : null - }) - } - - result = result.findAll { it != null } - result.sort() + date = getFirst(date) + return removeMilliseconds(date) } def getPropertyFromSurvey(Map surveyData, String path) { @@ -107,21 +70,13 @@ class ParatooProtocolConfig { return null } - def date - if (usesPlotLayout) { - def dates = getDatesFromObservation(surveyData) - date = dates ? DateUtil.format(dates.last()) : null - return date + def date = getProperty(surveyData, endDatePath) + if (!date) { + date = getPropertyFromSurvey(surveyData, endDatePath) } - else { - date = getProperty(surveyData, endDatePath) - if (!date) { - date = getPropertyFromSurvey(surveyData, endDatePath) - } - date = getFirst(date) - return removeMilliseconds(date) - } + date = getFirst(date) + return removeMilliseconds(date) } Map getSurveyId(Map surveyData) { diff --git a/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy b/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy index f79449bdf..5cf4b473b 100644 --- a/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy +++ b/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy @@ -197,7 +197,11 @@ class ParatooServiceSpec extends MongoSpec implements ServiceUnitTest> [projectId: projectId, custom: [dataSets: [dataSet]]] 1 * projectService.update([custom: [dataSets: [expectedDataSetAsync]]], 'p1', false) >> [status: 'ok'] 1 * projectService.update([custom: [dataSets: [expectedDataSetSync]]], 'p1', false) >> [status: 'ok'] - 1 * activityService.create(_) >> [activityId: '123'] + 1 * activityService.create({ + it.startDate == "2023-09-01T00:00:00Z" && it.endDate == "2023-09-01T00:00:00Z" && + it.plannedStartDate == "2023-09-01T00:00:00Z" && it.plannedEndDate == "2023-09-01T00:00:00Z" && + it.externalIds[0].externalId == "d1" && it.externalIds[0].idType == ExternalId.IdType.MONITOR_MINTED_COLLECTION_ID + }) >> [activityId: '123'] 1 * activityService.delete("123", true) >> [status: 'ok'] 1 * recordService.getAllByActivity('123') >> [] 1 * settingService.getSetting('paratoo.surveyData.mapping') >> { @@ -307,7 +311,11 @@ class ParatooServiceSpec extends MongoSpec implements ServiceUnitTest> [status: 'ok'] 2 * projectService.get(projectId) >> [projectId: projectId, custom: [dataSets: [dataSet]]] 1 * siteService.create(_) >> { site = it[0]; [siteId: 's1'] } - 1 * activityService.create(_) >> [activityId: '123'] + 1 * activityService.create({ + it.startDate == "2023-09-22T00:59:47Z" && it.endDate == "2023-09-23T00:59:47Z" && + it.plannedStartDate == "2023-09-22T00:59:47Z" && it.plannedEndDate == "2023-09-23T00:59:47Z" && + it.externalIds[0].externalId == "d1" && it.externalIds[0].idType == ExternalId.IdType.MONITOR_MINTED_COLLECTION_ID + }) >> [activityId: '123'] 1 * recordService.getAllByActivity('123') >> [] 1 * settingService.getSetting('paratoo.surveyData.mapping') >> { (["guid-3": [ diff --git a/src/test/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfigSpec.groovy b/src/test/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfigSpec.groovy index dbfbf87fa..9b2560aac 100644 --- a/src/test/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfigSpec.groovy +++ b/src/test/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfigSpec.groovy @@ -100,9 +100,7 @@ class ParatooProtocolConfigSpec extends Specification { Map observation = apiOutput.collections Map floristicsSurveyConfig = [ apiEndpoint:'floristics-veg-survey-lites', - usesPlotLayout:true, - startDatePath: 'plot_visit.start_date', - endDatePath: 'plot_visit.end_date' + usesPlotLayout:true ] ParatooProtocolConfig config = new ParatooProtocolConfig(floristicsSurveyConfig) config.setSurveyId(ParatooCollectionId.fromMap([survey_metadata: apiOutput.survey_metadata])) @@ -134,8 +132,8 @@ class ParatooProtocolConfigSpec extends Specification { transformData(observation, activityForm, config) then: - config.getStartDate(observation) == "2024-04-08T01:23:28Z" - config.getEndDate(observation) == "2024-04-10T01:23:28Z" + config.getStartDate(observation) == "2022-09-21T01:55:44Z" + config.getEndDate(observation) == "2022-09-21T01:55:44Z" config.getGeoJson(observation) == [type: "Feature", geometry: [type: "Polygon", coordinates: [[[152.880694, -27.388252], [152.880651, -27.388336], [152.880518, -27.388483], [152.880389, -27.388611], [152.88028, -27.388749], [152.880154, -27.388903], [152.880835, -27.389463], [152.880644, -27.389366], [152.880525, -27.389248], [152.88035, -27.389158], [152.880195, -27.389021], [152.880195, -27.389373], [152.880797, -27.388316], [152.881448, -27.388909], [152.881503, -27.388821], [152.881422, -27.388766], [152.881263, -27.388644], [152.881107, -27.388549], [152.880939, -27.388445], [152.881314, -27.389035], [152.88122, -27.389208], [152.881089, -27.389343], [152.880973, -27.389472], [152.880916, -27.389553], [152.880694, -27.388252]]]], properties: [name: "QDASEQ0001 - Control (100 x 100)", externalId: 1, description: "QDASEQ0001 - Control (100 x 100)", notes: "some comment"]] } @@ -144,9 +142,7 @@ class ParatooProtocolConfigSpec extends Specification { Map surveyData = readSurveyData('basalAreaDbhReverseLookup') Map basalAreaDbhMeasureSurveyConfig = [ apiEndpoint:'basal-area-dbh-measure-surveys', - usesPlotLayout:true, - startDatePath: 'start_date', - endDatePath: 'start_date', + usesPlotLayout:true ] ParatooProtocolConfig config = new ParatooProtocolConfig(basalAreaDbhMeasureSurveyConfig) config.setSurveyId(ParatooCollectionId.fromMap([survey_metadata: surveyData.survey_metadata])) @@ -185,8 +181,8 @@ class ParatooProtocolConfigSpec extends Specification { transformData(observation, activityForm, config) expect: - config.getStartDate(observation) == "2024-03-28T03:17:01Z" - config.getEndDate(observation) == "2024-03-28T03:17:01Z" + config.getStartDate(observation) == "2023-09-22T00:59:47Z" + config.getEndDate(observation) == "2023-09-23T00:59:47Z" config.getGeoJson(observation) == [ type : "Feature", geometry : [ @@ -211,9 +207,7 @@ class ParatooProtocolConfigSpec extends Specification { Map opportunisticSurveyConfig = [ apiEndpoint : 'opportunistic-surveys', usesPlotLayout: false, - geometryType : 'Point', - startDatePath : 'start_date_time', - endDatePath : 'end_date_time' + geometryType : 'Point' ] ActivityForm activityForm = new ActivityForm( name: "aParatooForm 1", diff --git a/src/test/resources/paratoo/basalAreaDbhReverseLookup.json b/src/test/resources/paratoo/basalAreaDbhReverseLookup.json index 27145c2ab..15ab3ed80 100644 --- a/src/test/resources/paratoo/basalAreaDbhReverseLookup.json +++ b/src/test/resources/paratoo/basalAreaDbhReverseLookup.json @@ -41,7 +41,8 @@ "system_org": "MERIT" } }, - "start_date": "2023-09-22T00:59:47.807Z", + "start_date_time": "2023-09-22T00:59:47.807Z", + "end_date_time": "2023-09-23T00:59:47.807Z", "createdAt": "2023-09-22T01:05:19.981Z", "updatedAt": "2023-09-22T01:05:19.981Z", "plot_size": { From 3c2b83d23febe3d3677297fe5aa69ed1c1247213 Mon Sep 17 00:00:00 2001 From: chrisala Date: Thu, 2 May 2024 12:02:33 +1000 Subject: [PATCH 05/44] Back to 4.6-SNAPSHOT #927 --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 21e18e5cc..56e413bff 100644 --- a/build.gradle +++ b/build.gradle @@ -20,7 +20,7 @@ plugins { id "com.gorylenko.gradle-git-properties" version "2.4.1" } -version "4.6-DATES-SNAPSHOT" +version "4.6-SNAPSHOT" group "au.org.ala" description "Ecodata" From 470974152ff86f893682ef72bad839c0296da75e Mon Sep 17 00:00:00 2001 From: chrisala Date: Thu, 2 May 2024 14:38:10 +1000 Subject: [PATCH 06/44] Allow updates of single data sets #934 --- .../ecodata/DataSetSummaryController.groovy | 30 +++++++ .../au/org/ala/ecodata/ParatooService.groovy | 32 ++----- .../au/org/ala/ecodata/ProjectService.groovy | 83 ++++++++++++++----- .../DataSetSummaryControllerSpec.groovy | 44 ++++++++++ .../org/ala/ecodata/ParatooServiceSpec.groovy | 16 ++-- .../org/ala/ecodata/ProjectServiceSpec.groovy | 45 ++++++++++ 6 files changed, 193 insertions(+), 57 deletions(-) create mode 100644 grails-app/controllers/au/org/ala/ecodata/DataSetSummaryController.groovy create mode 100644 src/test/groovy/au/org/ala/ecodata/DataSetSummaryControllerSpec.groovy diff --git a/grails-app/controllers/au/org/ala/ecodata/DataSetSummaryController.groovy b/grails-app/controllers/au/org/ala/ecodata/DataSetSummaryController.groovy new file mode 100644 index 000000000..d6572b73f --- /dev/null +++ b/grails-app/controllers/au/org/ala/ecodata/DataSetSummaryController.groovy @@ -0,0 +1,30 @@ +package au.org.ala.ecodata + +import org.apache.http.HttpStatus + +class DataSetSummaryController { + + static responseFormats = ['json', 'xml'] + static allowedMethods = [update:['POST', 'PUT']] + + ProjectService projectService + + /** Updates a single dataset for a project */ + def update(String projectId) { + Map dataSet = request.JSON + + if (!projectId && !dataSet.projectId) { + respond status: 400, message: "projectId is required" + return + } + + projectId = projectId || dataSet.projectId + + if (dataSet.projectId && dataSet.projectId != projectId) { + respond status: HttpStatus.SC_BAD_REQUEST, message: "projectId must match the data set projectId" + return + } + + respond projectService.updateDataSet(projectId, dataSet) + } +} diff --git a/grails-app/services/au/org/ala/ecodata/ParatooService.groovy b/grails-app/services/au/org/ala/ecodata/ParatooService.groovy index 267c868e4..0e6add911 100644 --- a/grails-app/services/au/org/ala/ecodata/ParatooService.groovy +++ b/grails-app/services/au/org/ala/ecodata/ParatooService.groovy @@ -18,7 +18,7 @@ import static grails.async.Promises.task */ @Slf4j class ParatooService { - static final Object LOCK = new Object() + static final String DATASET_DATABASE_TABLE = 'Database Table' static final int PARATOO_MAX_RETRIES = 3 static final String PARATOO_PROTOCOL_PATH = '/protocols' @@ -186,18 +186,7 @@ class ParatooService { dataSet.orgMintedIdentifier = paratooCollectionId.encodeAsOrgMintedIdentifier() log.info "Minting identifier for Monitor collection: ${paratooCollectionId}: ${dataSet.orgMintedIdentifier}" - Map result - synchronized (LOCK) { - Map latestProject = projectService.get(projectId) - if (!latestProject.custom) { - latestProject.custom = [:] - } - if (!latestProject.custom.dataSets) { - latestProject.custom.dataSets = [] - } - latestProject.custom.dataSets << dataSet - result = projectService.update([custom: latestProject.custom], projectId, false) - } + Map result = projectService.updateDataSet(projectId, dataSet) if (!result.error) { result.orgMintedIdentifier = dataSet.orgMintedIdentifier @@ -239,18 +228,12 @@ class ParatooService { log.error("An error occurred feching ${collection.orgMintedUUID}: ${e.message}", e) userService.clearCurrentUser() } - + promise.onComplete { Map result -> userService.clearCurrentUser() } - def result - synchronized (LOCK) { - Map latestProject = projectService.get(project.id) - Map latestDataSet = latestProject.custom?.dataSets?.find { it.dataSetId == collection.orgMintedUUID } - latestDataSet.putAll(dataSet) - result = projectService.update([custom: latestProject.custom], project.id, false) - } + def result = projectService.updateDataSet(project.id, dataSet) [updateResult: result, promise: promise] } @@ -320,12 +303,7 @@ class ParatooService { dataSet.areSpeciesRecorded = records?.size() > 0 dataSet.activityId = activityId - synchronized (LOCK) { - Map latestProject = projectService.get(project.project.projectId) - Map latestDataSet = latestProject.custom?.dataSets?.find { it.dataSetId == collection.orgMintedUUID } - latestDataSet.putAll(dataSet) - projectService.update([custom: latestProject.custom], project.id, false) - } + projectService.updateDataSet(project.id, dataSet) } } } diff --git a/grails-app/services/au/org/ala/ecodata/ProjectService.groovy b/grails-app/services/au/org/ala/ecodata/ProjectService.groovy index b1ee890db..ec5467880 100644 --- a/grails-app/services/au/org/ala/ecodata/ProjectService.groovy +++ b/grails-app/services/au/org/ala/ecodata/ProjectService.groovy @@ -26,6 +26,11 @@ class ProjectService { static final ENHANCED = 'enhanced' static final PRIVATE_SITES_REMOVED = 'privatesitesremoved' + /** A Map containing a per-project lock for synchronizing locks for updates. The purpose of this + * is to support concurrent edits on different project data set summaries which are currently modelled as + * an embedded array but can be added and updated by both the UI and the Monitor (Parataoo) application API */ + static final Map PROJECT_UPDATE_LOCKS = Collections.synchronizedMap([:].withDefault{ new Object() }) + GrailsApplication grailsApplication MessageSource messageSource SessionLocaleResolver localeResolver @@ -51,6 +56,8 @@ class ProjectService { grailsApplication.mainContext.commonService }*/ + + def getBrief(listOfIds, version = null) { if (listOfIds) { if (version) { @@ -469,30 +476,32 @@ class ProjectService { } def update(Map props, String id, Boolean shouldUpdateCollectory = true) { - Project project = Project.findByProjectId(id) - if (project) { - // retrieve any project activities associated with the project - List projectActivities = projectActivityService.getAllByProject(id) - props = includeProjectFundings(props) - props = includeProjectActivities(props, projectActivities) + synchronized (PROJECT_UPDATE_LOCKS.get(id)) { + Project project = Project.findByProjectId(id) + if (project) { + // retrieve any project activities associated with the project + List projectActivities = projectActivityService.getAllByProject(id) + props = includeProjectFundings(props) + props = includeProjectActivities(props, projectActivities) - try { - bindEmbeddedProperties(project, props) - commonService.updateProperties(project, props) - if (shouldUpdateCollectory) { - updateCollectoryLinkForProject(project, props) + try { + bindEmbeddedProperties(project, props) + commonService.updateProperties(project, props) + if (shouldUpdateCollectory) { + updateCollectoryLinkForProject(project, props) + } + return [status: 'ok'] + } catch (Exception e) { + Project.withSession { session -> session.clear() } + def error = "Error updating project ${id} - ${e.message}" + log.error error, e + return [status: 'error', error: error] } - return [status: 'ok'] - } catch (Exception e) { - Project.withSession { session -> session.clear() } - def error = "Error updating project ${id} - ${e.message}" - log.error error, e + } else { + def error = "Error updating project - no such id ${id}" + log.error error return [status: 'error', error: error] } - } else { - def error = "Error updating project - no such id ${id}" - log.error error - return [status: 'error', error: error] } } @@ -1054,4 +1063,38 @@ class ProjectService { records } + /** + * Updates a single data set associated with a project. Because the datasets are stored as an embedded + * array in the Project collection, this method is synchronized on the project to avoid concurrent updates to + * different data sets overwriting each other. + * Due to the way it's been modelled as an embedded array, the client is allowed to supply a dataSetId + * when creating a new data set (e.g. a data set created by a submission from the Monitor app uses the + * submissionId as the dataSetId). + * @param projectId The project to update + * @param dataSet the data set to update. + * @return + */ + Map updateDataSet(String projectId, Map dataSet) { + synchronized (PROJECT_UPDATE_LOCKS.get(projectId)) { + Project project = Project.findByProjectId(projectId) + + if (!dataSet.dataSetId) { + dataSet.dataSetId = Identifiers.getNew(true, '') + } + Map matchingDataSet = project.custom?.dataSets?.find { it.dataSetId == dataSet.dataSetId } + if (matchingDataSet) { + matchingDataSet.putAll(dataSet) + } else { + if (!project.custom) { + project.custom = [:] + } + if (!project.custom?.dataSets) { + project.custom.dataSets = [] + } + project.custom.dataSets.add(dataSet) + } + update([custom: project.custom], project.projectId, false) + } + } + } \ No newline at end of file diff --git a/src/test/groovy/au/org/ala/ecodata/DataSetSummaryControllerSpec.groovy b/src/test/groovy/au/org/ala/ecodata/DataSetSummaryControllerSpec.groovy new file mode 100644 index 000000000..f838f87dd --- /dev/null +++ b/src/test/groovy/au/org/ala/ecodata/DataSetSummaryControllerSpec.groovy @@ -0,0 +1,44 @@ +package au.org.ala.ecodata + +import grails.testing.web.controllers.ControllerUnitTest +import org.apache.http.HttpStatus +import spock.lang.Specification + +class DataSetSummaryControllerSpec extends Specification implements ControllerUnitTest { + + ProjectService projectService = Mock(ProjectService) + def setup() { + controller.projectService = projectService + } + + def cleanup() { + } + + void "The update method delegates to the projectService"() { + setup: + String projectId = 'p1' + Map dataSetSummary = [dataSetId:'d1', name:'Data set 1'] + + when: + request.json = dataSetSummary + controller.update(projectId) + + then: + 1 * projectService.updateDataSet(projectId, dataSetSummary) >> [status:'ok'] + response.json == ['status':'ok'] + + } + + void "A project id must be specified either in the path or as part of the data set summary"() { + setup: + Map dataSetSummary = [dataSetId: 'd1', name: 'Data set 1'] + + when: + request.json = dataSetSummary + controller.update() + + then: + 0 * projectService.updateDataSet(_, _) + response.status == HttpStatus.SC_BAD_REQUEST + } +} diff --git a/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy b/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy index d4ff42c29..180c1350f 100644 --- a/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy +++ b/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy @@ -138,19 +138,17 @@ class ParatooServiceSpec extends MongoSpec implements ServiceUnitTest> project - 1 * projectService.update(_, projectId, false) >> { data, pId, updateCollectory -> - Map dataSet = data.custom.dataSets[1] // The stubbed project already has a dataSet, so the new one will be index=1 + 1 * projectService.updateDataSet(projectId, _) >> { pId, dataSet -> + pId == projectId assert dataSet.surveyId != null assert dataSet.surveyId.eventTime != null assert dataSet.surveyId.userId == 'org1' @@ -194,9 +192,8 @@ class ParatooServiceSpec extends MongoSpec implements ServiceUnitTest> [resp: [collections: ["coarse-woody-debris-survey": [uuid: "1", createdAt: "2023-09-01T00:00:00.123Z", start_date_time: "2023-09-01T00:00:00.123Z", end_date_time: "2023-09-01T00:00:00.123Z"]]]] 1 * tokenService.getAuthToken(true) >> Mock(AccessToken) - 2 * projectService.get(projectId) >> [projectId: projectId, custom: [dataSets: [dataSet]]] - 1 * projectService.update([custom: [dataSets: [expectedDataSetAsync]]], 'p1', false) >> [status: 'ok'] - 1 * projectService.update([custom: [dataSets: [expectedDataSetSync]]], 'p1', false) >> [status: 'ok'] + 1 * projectService.updateDataSet(projectId, expectedDataSetAsync) >> [status: 'ok'] + 1 * projectService.updateDataSet(projectId, expectedDataSetSync) >> [status: 'ok'] 1 * activityService.create({ it.startDate == "2023-09-01T00:00:00Z" && it.endDate == "2023-09-01T00:00:00Z" && it.plannedStartDate == "2023-09-01T00:00:00Z" && it.plannedEndDate == "2023-09-01T00:00:00Z" && @@ -310,8 +307,7 @@ class ParatooServiceSpec extends MongoSpec implements ServiceUnitTest> [resp: surveyData] 1 * tokenService.getAuthToken(true) >> Mock(AccessToken) - 2 * projectService.update(_, projectId, false) >> [status: 'ok'] - 2 * projectService.get(projectId) >> [projectId: projectId, custom: [dataSets: [dataSet]]] + 2 * projectService.updateDataSet(projectId, _) >> [status: 'ok'] 1 * siteService.create(_) >> { site = it[0]; [siteId: 's1'] } 1 * activityService.create({ it.startDate == "2023-09-22T00:59:47Z" && it.endDate == "2023-09-23T00:59:47Z" && diff --git a/src/test/groovy/au/org/ala/ecodata/ProjectServiceSpec.groovy b/src/test/groovy/au/org/ala/ecodata/ProjectServiceSpec.groovy index f73b83a3e..dd9b6d1fa 100644 --- a/src/test/groovy/au/org/ala/ecodata/ProjectServiceSpec.groovy +++ b/src/test/groovy/au/org/ala/ecodata/ProjectServiceSpec.groovy @@ -761,4 +761,49 @@ class ProjectServiceSpec extends MongoSpec implements ServiceUnitTest Date: Thu, 2 May 2024 15:04:59 +1000 Subject: [PATCH 07/44] Fixed controller/test #934 --- .../org/ala/ecodata/DataSetSummaryController.groovy | 11 ++++++----- .../ala/ecodata/DataSetSummaryControllerSpec.groovy | 2 ++ 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/grails-app/controllers/au/org/ala/ecodata/DataSetSummaryController.groovy b/grails-app/controllers/au/org/ala/ecodata/DataSetSummaryController.groovy index d6572b73f..666c8f3bb 100644 --- a/grails-app/controllers/au/org/ala/ecodata/DataSetSummaryController.groovy +++ b/grails-app/controllers/au/org/ala/ecodata/DataSetSummaryController.groovy @@ -13,15 +13,16 @@ class DataSetSummaryController { def update(String projectId) { Map dataSet = request.JSON - if (!projectId && !dataSet.projectId) { - respond status: 400, message: "projectId is required" + if (!projectId) { + projectId = dataSet.projectId + } + if (!projectId) { + render status: HttpStatus.SC_BAD_REQUEST, text: "projectId is required" return } - projectId = projectId || dataSet.projectId - if (dataSet.projectId && dataSet.projectId != projectId) { - respond status: HttpStatus.SC_BAD_REQUEST, message: "projectId must match the data set projectId" + render status: HttpStatus.SC_BAD_REQUEST, text: "projectId must match the data set projectId" return } diff --git a/src/test/groovy/au/org/ala/ecodata/DataSetSummaryControllerSpec.groovy b/src/test/groovy/au/org/ala/ecodata/DataSetSummaryControllerSpec.groovy index f838f87dd..eeee899a8 100644 --- a/src/test/groovy/au/org/ala/ecodata/DataSetSummaryControllerSpec.groovy +++ b/src/test/groovy/au/org/ala/ecodata/DataSetSummaryControllerSpec.groovy @@ -20,6 +20,7 @@ class DataSetSummaryControllerSpec extends Specification implements ControllerUn Map dataSetSummary = [dataSetId:'d1', name:'Data set 1'] when: + request.method = 'POST' request.json = dataSetSummary controller.update(projectId) @@ -34,6 +35,7 @@ class DataSetSummaryControllerSpec extends Specification implements ControllerUn Map dataSetSummary = [dataSetId: 'd1', name: 'Data set 1'] when: + request.method = 'POST' request.json = dataSetSummary controller.update() From f3a59ff4bbe25b303115d95d70d341311ce97e31 Mon Sep 17 00:00:00 2001 From: chrisala Date: Fri, 3 May 2024 09:59:40 +1000 Subject: [PATCH 08/44] Added a delete data set operation #934 --- .../ecodata/DataSetSummaryController.groovy | 14 ++++++++---- .../au/org/ala/ecodata/UrlMappings.groovy | 4 ++++ .../au/org/ala/ecodata/ProjectService.groovy | 18 ++++++++++++++- .../DataSetSummaryControllerSpec.groovy | 14 ++++++++++++ .../org/ala/ecodata/ProjectServiceSpec.groovy | 22 +++++++++++++++++++ 5 files changed, 67 insertions(+), 5 deletions(-) diff --git a/grails-app/controllers/au/org/ala/ecodata/DataSetSummaryController.groovy b/grails-app/controllers/au/org/ala/ecodata/DataSetSummaryController.groovy index 666c8f3bb..7f5b629e2 100644 --- a/grails-app/controllers/au/org/ala/ecodata/DataSetSummaryController.groovy +++ b/grails-app/controllers/au/org/ala/ecodata/DataSetSummaryController.groovy @@ -5,17 +5,15 @@ import org.apache.http.HttpStatus class DataSetSummaryController { static responseFormats = ['json', 'xml'] - static allowedMethods = [update:['POST', 'PUT']] + static allowedMethods = [update:['POST', 'PUT'], delete:'DELETE'] ProjectService projectService /** Updates a single dataset for a project */ def update(String projectId) { Map dataSet = request.JSON + projectId = projectId ?: dataSet.projectId - if (!projectId) { - projectId = dataSet.projectId - } if (!projectId) { render status: HttpStatus.SC_BAD_REQUEST, text: "projectId is required" return @@ -28,4 +26,12 @@ class DataSetSummaryController { respond projectService.updateDataSet(projectId, dataSet) } + + def delete(String projectId, String dataSetId) { + if (!projectId || !dataSetId) { + render status: HttpStatus.SC_BAD_REQUEST, text: "projectId and dataSetId are required" + return + } + respond projectService.deleteDataSet(projectId, dataSetId) + } } diff --git a/grails-app/controllers/au/org/ala/ecodata/UrlMappings.groovy b/grails-app/controllers/au/org/ala/ecodata/UrlMappings.groovy index 7f85d0b43..4e2f62986 100644 --- a/grails-app/controllers/au/org/ala/ecodata/UrlMappings.groovy +++ b/grails-app/controllers/au/org/ala/ecodata/UrlMappings.groovy @@ -196,6 +196,10 @@ class UrlMappings { "/ws/project/getDefaultFacets"(controller: "project", action: "getDefaultFacets") "/ws/project/$projectId/dataSet/$dataSetId/records"(controller: "project", action: "fetchDataSetRecords") "/ws/admin/initiateSpeciesRematch"(controller: "admin", action: "initiateSpeciesRematch") + "/ws/dataSetSummary/$projectId/$dataSetId?"(controller :'dataSetSummary') { + + action = [POST:'update', PUT:'update', DELETE:'delete'] + } "/ws/document/download"(controller:"document", action:"download") diff --git a/grails-app/services/au/org/ala/ecodata/ProjectService.groovy b/grails-app/services/au/org/ala/ecodata/ProjectService.groovy index ec5467880..41b8b09a0 100644 --- a/grails-app/services/au/org/ala/ecodata/ProjectService.groovy +++ b/grails-app/services/au/org/ala/ecodata/ProjectService.groovy @@ -1077,7 +1077,9 @@ class ProjectService { Map updateDataSet(String projectId, Map dataSet) { synchronized (PROJECT_UPDATE_LOCKS.get(projectId)) { Project project = Project.findByProjectId(projectId) - + if (!project) { + return [status: 'error', error: "No project exists with projectId=${projectId}"] + } if (!dataSet.dataSetId) { dataSet.dataSetId = Identifiers.getNew(true, '') } @@ -1097,4 +1099,18 @@ class ProjectService { } } + Map deleteDataSet(String projectId, String dataSetId) { + synchronized (PROJECT_UPDATE_LOCKS.get(projectId)) { + Project project = Project.findByProjectId(projectId) + + boolean foundMatchingDataSet = project?.custom?.dataSets?.removeAll { it.dataSetId == dataSetId } + if (!foundMatchingDataSet) { + return [status: 'error', error: 'No such data set'] + } + else { + update([custom: project.custom], project.projectId, false) + } + } + } + } \ No newline at end of file diff --git a/src/test/groovy/au/org/ala/ecodata/DataSetSummaryControllerSpec.groovy b/src/test/groovy/au/org/ala/ecodata/DataSetSummaryControllerSpec.groovy index eeee899a8..46d7fc1e1 100644 --- a/src/test/groovy/au/org/ala/ecodata/DataSetSummaryControllerSpec.groovy +++ b/src/test/groovy/au/org/ala/ecodata/DataSetSummaryControllerSpec.groovy @@ -43,4 +43,18 @@ class DataSetSummaryControllerSpec extends Specification implements ControllerUn 0 * projectService.updateDataSet(_, _) response.status == HttpStatus.SC_BAD_REQUEST } + + void "The delete method delegates to the projectService"() { + setup: + String projectId = 'p1' + String dataSetSummaryId = 'd1' + + when: + request.method = 'DELETE' + controller.delete(projectId, dataSetSummaryId) + + then: + 1 * projectService.deleteDataSet(projectId, dataSetSummaryId) >> [status:'ok'] + response.json == ['status':'ok'] + } } diff --git a/src/test/groovy/au/org/ala/ecodata/ProjectServiceSpec.groovy b/src/test/groovy/au/org/ala/ecodata/ProjectServiceSpec.groovy index dd9b6d1fa..856ee7b16 100644 --- a/src/test/groovy/au/org/ala/ecodata/ProjectServiceSpec.groovy +++ b/src/test/groovy/au/org/ala/ecodata/ProjectServiceSpec.groovy @@ -806,4 +806,26 @@ class ProjectServiceSpec extends MongoSpec implements ServiceUnitTest Date: Sat, 4 May 2024 11:22:38 +1000 Subject: [PATCH 09/44] Merge Project custom property instead of replace in update method #934 --- .../au/org/ala/ecodata/ProjectService.groovy | 5 +++++ .../org/ala/ecodata/ProjectServiceSpec.groovy | 22 +++++++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/grails-app/services/au/org/ala/ecodata/ProjectService.groovy b/grails-app/services/au/org/ala/ecodata/ProjectService.groovy index 41b8b09a0..8e6840a47 100644 --- a/grails-app/services/au/org/ala/ecodata/ProjectService.groovy +++ b/grails-app/services/au/org/ala/ecodata/ProjectService.groovy @@ -485,6 +485,11 @@ class ProjectService { props = includeProjectActivities(props, projectActivities) try { + // Custom currently holds keys "details" and "dataSets". Only update the "custom" properties + // that are supplied in the update, leaving the others intact. + if (project.custom && props.custom) { + project.custom.putAll(props.remove('custom')) + } bindEmbeddedProperties(project, props) commonService.updateProperties(project, props) if (shouldUpdateCollectory) { diff --git a/src/test/groovy/au/org/ala/ecodata/ProjectServiceSpec.groovy b/src/test/groovy/au/org/ala/ecodata/ProjectServiceSpec.groovy index 856ee7b16..a5b01f8e7 100644 --- a/src/test/groovy/au/org/ala/ecodata/ProjectServiceSpec.groovy +++ b/src/test/groovy/au/org/ala/ecodata/ProjectServiceSpec.groovy @@ -828,4 +828,26 @@ class ProjectServiceSpec extends MongoSpec implements ServiceUnitTest Date: Mon, 6 May 2024 14:58:39 +1000 Subject: [PATCH 10/44] Added bulk update method #934 --- .../ecodata/DataSetSummaryController.groovy | 29 +++++++++++- .../au/org/ala/ecodata/UrlMappings.groovy | 2 + .../au/org/ala/ecodata/ProjectService.groovy | 45 ++++++++++++++----- .../DataSetSummaryControllerSpec.groovy | 15 +++++++ 4 files changed, 78 insertions(+), 13 deletions(-) diff --git a/grails-app/controllers/au/org/ala/ecodata/DataSetSummaryController.groovy b/grails-app/controllers/au/org/ala/ecodata/DataSetSummaryController.groovy index 7f5b629e2..899399210 100644 --- a/grails-app/controllers/au/org/ala/ecodata/DataSetSummaryController.groovy +++ b/grails-app/controllers/au/org/ala/ecodata/DataSetSummaryController.groovy @@ -5,7 +5,7 @@ import org.apache.http.HttpStatus class DataSetSummaryController { static responseFormats = ['json', 'xml'] - static allowedMethods = [update:['POST', 'PUT'], delete:'DELETE'] + static allowedMethods = [update:['POST', 'PUT'], delete:'DELETE', bulkUpdate: 'POST'] ProjectService projectService @@ -27,6 +27,33 @@ class DataSetSummaryController { respond projectService.updateDataSet(projectId, dataSet) } + /** + * Updates multiple data sets for a project. + * This endpoint exists to support the use case of associating multiple data sets with a + * report and updating their publicationStatus when the report is submitted/approved. + * + * This method expects the projectId to be supplied via the URL and the data sets to be supplied in the request + * body as a JSON object with key="dataSets" and value=List of data sets. + */ + def bulkUpdate(String projectId) { + Map postBody = request.JSON + List dataSets = postBody?.dataSets + + if (!projectId) { + render status: HttpStatus.SC_BAD_REQUEST, text: "projectId is required" + return + } + + dataSets.each { Map dataSet -> + if (dataSet.projectId && dataSet.projectId != projectId) { + render status: HttpStatus.SC_BAD_REQUEST, text: "projectId must match the data set projectId" + return + } + } + + respond projectService.updateDataSets(projectId, dataSets) + } + def delete(String projectId, String dataSetId) { if (!projectId || !dataSetId) { render status: HttpStatus.SC_BAD_REQUEST, text: "projectId and dataSetId are required" diff --git a/grails-app/controllers/au/org/ala/ecodata/UrlMappings.groovy b/grails-app/controllers/au/org/ala/ecodata/UrlMappings.groovy index 4e2f62986..c759f7a07 100644 --- a/grails-app/controllers/au/org/ala/ecodata/UrlMappings.groovy +++ b/grails-app/controllers/au/org/ala/ecodata/UrlMappings.groovy @@ -201,6 +201,8 @@ class UrlMappings { action = [POST:'update', PUT:'update', DELETE:'delete'] } + "/ws/dataSetSummary/bulkUpdate/$projectId"(controller:'dataSetSummary', action:'bulkUpdate') + "/ws/document/download"(controller:"document", action:"download") "/ws/$controller/list"() { action = [GET:'list'] } diff --git a/grails-app/services/au/org/ala/ecodata/ProjectService.groovy b/grails-app/services/au/org/ala/ecodata/ProjectService.groovy index 8e6840a47..8fab9f617 100644 --- a/grails-app/services/au/org/ala/ecodata/ProjectService.groovy +++ b/grails-app/services/au/org/ala/ecodata/ProjectService.groovy @@ -1080,25 +1080,46 @@ class ProjectService { * @return */ Map updateDataSet(String projectId, Map dataSet) { + updateDataSets(projectId, [dataSet]) + } + + /** + * Updates multiple data sets associated with a project at the same time. This method exists to support + * the use case of associating multiple data sets with a report and updating their publicationStatus when + * the report is submitted/approved. + * + * Because the datasets are stored as an embedded + * array in the Project collection, this method is synchronized on the project to avoid concurrent updates to + * different data sets overwriting each other. + * Due to the way it's been modelled as an embedded array, the client is allowed to supply a dataSetId + * when creating a new data set (e.g. a data set created by a submission from the Monitor app uses the + * submissionId as the dataSetId). + * @param projectId The project to update + * @param dataSet the data sets to update. + * @return + */ + Map updateDataSets(String projectId, List dataSets) { synchronized (PROJECT_UPDATE_LOCKS.get(projectId)) { Project project = Project.findByProjectId(projectId) if (!project) { return [status: 'error', error: "No project exists with projectId=${projectId}"] } - if (!dataSet.dataSetId) { - dataSet.dataSetId = Identifiers.getNew(true, '') - } - Map matchingDataSet = project.custom?.dataSets?.find { it.dataSetId == dataSet.dataSetId } - if (matchingDataSet) { - matchingDataSet.putAll(dataSet) - } else { - if (!project.custom) { - project.custom = [:] + for (Map dataSet in dataSets) { + if (!dataSet.dataSetId) { + dataSet.dataSetId = Identifiers.getNew(true, '') } - if (!project.custom?.dataSets) { - project.custom.dataSets = [] + Map matchingDataSet = project.custom?.dataSets?.find { it.dataSetId == dataSet.dataSetId } + if (matchingDataSet) { + matchingDataSet.putAll(dataSet) + } else { + if (!project.custom) { + project.custom = [:] + } + if (!project.custom?.dataSets) { + project.custom.dataSets = [] + } + project.custom.dataSets.add(dataSet) } - project.custom.dataSets.add(dataSet) } update([custom: project.custom], project.projectId, false) } diff --git a/src/test/groovy/au/org/ala/ecodata/DataSetSummaryControllerSpec.groovy b/src/test/groovy/au/org/ala/ecodata/DataSetSummaryControllerSpec.groovy index 46d7fc1e1..9b9db1b28 100644 --- a/src/test/groovy/au/org/ala/ecodata/DataSetSummaryControllerSpec.groovy +++ b/src/test/groovy/au/org/ala/ecodata/DataSetSummaryControllerSpec.groovy @@ -57,4 +57,19 @@ class DataSetSummaryControllerSpec extends Specification implements ControllerUn 1 * projectService.deleteDataSet(projectId, dataSetSummaryId) >> [status:'ok'] response.json == ['status':'ok'] } + + void "The bulkUpdate method delegates to the projectService"() { + setup: + String projectId = 'p1' + Map postBody = [dataSets:[[dataSetId:'d1', name:'Data set 1']]] + + when: + request.method = 'POST' + request.json = postBody + controller.bulkUpdate(projectId) + + then: + 1 * projectService.updateDataSets(projectId, postBody.dataSets) >> [status:'ok'] + response.json == ['status':'ok'] + } } From a143e4ee526795b4278d47a3d3d0329f839ba2b9 Mon Sep 17 00:00:00 2001 From: chrisala Date: Tue, 7 May 2024 09:32:42 +1000 Subject: [PATCH 11/44] Addressed code review issue #934 --- .../ala/ecodata/DataSetSummaryController.groovy | 4 ++-- .../ecodata/DataSetSummaryControllerSpec.groovy | 15 +++++++++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/grails-app/controllers/au/org/ala/ecodata/DataSetSummaryController.groovy b/grails-app/controllers/au/org/ala/ecodata/DataSetSummaryController.groovy index 899399210..7a5d1a292 100644 --- a/grails-app/controllers/au/org/ala/ecodata/DataSetSummaryController.groovy +++ b/grails-app/controllers/au/org/ala/ecodata/DataSetSummaryController.groovy @@ -44,9 +44,9 @@ class DataSetSummaryController { return } - dataSets.each { Map dataSet -> + for (Map dataSet in dataSets) { if (dataSet.projectId && dataSet.projectId != projectId) { - render status: HttpStatus.SC_BAD_REQUEST, text: "projectId must match the data set projectId" + render status: HttpStatus.SC_BAD_REQUEST, text: "projectId must match the projectId in all supplied data sets" return } } diff --git a/src/test/groovy/au/org/ala/ecodata/DataSetSummaryControllerSpec.groovy b/src/test/groovy/au/org/ala/ecodata/DataSetSummaryControllerSpec.groovy index 9b9db1b28..d7dc36f56 100644 --- a/src/test/groovy/au/org/ala/ecodata/DataSetSummaryControllerSpec.groovy +++ b/src/test/groovy/au/org/ala/ecodata/DataSetSummaryControllerSpec.groovy @@ -72,4 +72,19 @@ class DataSetSummaryControllerSpec extends Specification implements ControllerUn 1 * projectService.updateDataSets(projectId, postBody.dataSets) >> [status:'ok'] response.json == ['status':'ok'] } + + void "If a projectId is present in a dataSet it much match the projectId parameter in bulkUpdate"() { + setup: + String projectId = 'p1' + Map postBody = [dataSets:[[dataSetId:'d1', name:'Data set 1', projectId:'p1'], [dataSetId:'d2', name:'Data set 2', projectId:'p2']]] + + when: + request.method = 'POST' + request.json = postBody + controller.bulkUpdate(projectId) + + then: + 0 * projectService.updateDataSets(_, _) + response.status == HttpStatus.SC_BAD_REQUEST + } } From fbdd2c6ed1a89d7bb4574ad3b695919005b95df3 Mon Sep 17 00:00:00 2001 From: chrisala Date: Mon, 13 May 2024 10:21:40 +1000 Subject: [PATCH 12/44] Ensure project is not cached in session #934 --- .../au/org/ala/ecodata/ProjectService.groovy | 36 +++++++++-------- .../org/ala/ecodata/ProjectServiceSpec.groovy | 40 +++++++++++++++++++ 2 files changed, 59 insertions(+), 17 deletions(-) diff --git a/grails-app/services/au/org/ala/ecodata/ProjectService.groovy b/grails-app/services/au/org/ala/ecodata/ProjectService.groovy index 8fab9f617..103cbb4e5 100644 --- a/grails-app/services/au/org/ala/ecodata/ProjectService.groovy +++ b/grails-app/services/au/org/ala/ecodata/ProjectService.groovy @@ -1100,28 +1100,30 @@ class ProjectService { */ Map updateDataSets(String projectId, List dataSets) { synchronized (PROJECT_UPDATE_LOCKS.get(projectId)) { - Project project = Project.findByProjectId(projectId) - if (!project) { - return [status: 'error', error: "No project exists with projectId=${projectId}"] - } - for (Map dataSet in dataSets) { - if (!dataSet.dataSetId) { - dataSet.dataSetId = Identifiers.getNew(true, '') + Project.withNewSession { // Ensure that the queried Project is not cached in the current session which can cause stale data + Project project = Project.findByProjectId(projectId) + if (!project) { + return [status: 'error', error: "No project exists with projectId=${projectId}"] } - Map matchingDataSet = project.custom?.dataSets?.find { it.dataSetId == dataSet.dataSetId } - if (matchingDataSet) { - matchingDataSet.putAll(dataSet) - } else { - if (!project.custom) { - project.custom = [:] + for (Map dataSet in dataSets) { + if (!dataSet.dataSetId) { + dataSet.dataSetId = Identifiers.getNew(true, '') } - if (!project.custom?.dataSets) { - project.custom.dataSets = [] + Map matchingDataSet = project.custom?.dataSets?.find { it.dataSetId == dataSet.dataSetId } + if (matchingDataSet) { + matchingDataSet.putAll(dataSet) + } else { + if (!project.custom) { + project.custom = [:] + } + if (!project.custom?.dataSets) { + project.custom.dataSets = [] + } + project.custom.dataSets.add(dataSet) } - project.custom.dataSets.add(dataSet) } + update([custom: project.custom], project.projectId, false) } - update([custom: project.custom], project.projectId, false) } } diff --git a/src/test/groovy/au/org/ala/ecodata/ProjectServiceSpec.groovy b/src/test/groovy/au/org/ala/ecodata/ProjectServiceSpec.groovy index a5b01f8e7..06d9b1e4b 100644 --- a/src/test/groovy/au/org/ala/ecodata/ProjectServiceSpec.groovy +++ b/src/test/groovy/au/org/ala/ecodata/ProjectServiceSpec.groovy @@ -9,6 +9,10 @@ import org.grails.web.converters.marshaller.json.CollectionMarshaller import org.grails.web.converters.marshaller.json.MapMarshaller import spock.lang.Ignore +import java.util.concurrent.Callable +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors + class ProjectServiceSpec extends MongoSpec implements ServiceUnitTest { ProjectActivityService projectActivityServiceStub = Stub(ProjectActivityService) @@ -850,4 +854,40 @@ class ProjectServiceSpec extends MongoSpec implements ServiceUnitTest Date: Mon, 13 May 2024 10:29:54 +1000 Subject: [PATCH 13/44] Github actions should build release branches #934 --- .github/workflows/build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 3849705ae..f0ba78b6d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -8,6 +8,7 @@ on: - master - feature/** - hotfix/** + - release/** env: TZ: Australia/Canberra From f86a80e76aeb3726085e0c54a33e47b834962950 Mon Sep 17 00:00:00 2001 From: chrisala Date: Mon, 13 May 2024 10:56:08 +1000 Subject: [PATCH 14/44] Update tests to account for different session management #934 --- .../org/ala/ecodata/ProjectServiceSpec.groovy | 22 ++++++++++++++----- 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/src/test/groovy/au/org/ala/ecodata/ProjectServiceSpec.groovy b/src/test/groovy/au/org/ala/ecodata/ProjectServiceSpec.groovy index 06d9b1e4b..3b379e926 100644 --- a/src/test/groovy/au/org/ala/ecodata/ProjectServiceSpec.groovy +++ b/src/test/groovy/au/org/ala/ecodata/ProjectServiceSpec.groovy @@ -770,13 +770,19 @@ class ProjectServiceSpec extends MongoSpec implements ServiceUnitTest Date: Tue, 14 May 2024 11:39:22 +1000 Subject: [PATCH 15/44] Update the data set name after more information is available #942 --- .../au/org/ala/ecodata/ParatooService.groovy | 36 ++++++++++++++----- .../groovy/au/org/ala/ecodata/DateUtil.groovy | 5 +++ .../org/ala/ecodata/ParatooServiceSpec.groovy | 11 +++++- 3 files changed, 43 insertions(+), 9 deletions(-) diff --git a/grails-app/services/au/org/ala/ecodata/ParatooService.groovy b/grails-app/services/au/org/ala/ecodata/ParatooService.groovy index 0e6add911..7a0bd4ef3 100644 --- a/grails-app/services/au/org/ala/ecodata/ParatooService.groovy +++ b/grails-app/services/au/org/ala/ecodata/ParatooService.groovy @@ -277,8 +277,11 @@ class ParatooService { surveyDataAndObservations = recursivelyTransformData(form.sections[0].template.dataModel, surveyDataAndObservations, form.name, 1, config) // If we are unable to create a site, null will be returned - assigning a null siteId is valid. + String siteName = null if (!dataSet.siteId) { - dataSet.siteId = createSiteFromSurveyData(surveyDataAndObservations, collection, surveyId, project.project, config, form) + Map site = createSiteFromSurveyData(surveyDataAndObservations, collection, surveyId, project.project, config, form) + dataSet.siteId = site.siteId + siteName = site.name } // plot layout is of type geoMap. Therefore, expects a site id. @@ -290,6 +293,9 @@ class ParatooService { dataSet.endDate = config.getEndDate(surveyDataAndObservations) dataSet.format = DATASET_DATABASE_TABLE dataSet.sizeUnknown = true + // Update the data set name as the information supplied during /mint-identifier isn't enough + // to ensure uniqueness + dataSet.name = buildUpdatedDataSetSummaryName(siteName, dataSet.startDate, dataSet.endDate, form.name, surveyId) // Delete previously created activity so that duplicate species records are not created. // Updating existing activity will also create duplicates since it relies on outputSpeciesId to determine @@ -308,6 +314,23 @@ class ParatooService { } } + protected static String buildUpdatedDataSetSummaryName(String siteName, String startDate, String endDate, String protocolName, ParatooCollectionId surveyId) { + String name = protocolName + if (siteName) { + name += " (" + siteName + ")" + } + if (startDate && endDate && startDate != endDate) { + name += " - " + DateUtil.formatAsDisplayDateTime(startDate) + " to " + DateUtil.formatAsDisplayDateTime(endDate) + } + else if (startDate) { + name += " - " +DateUtil.formatAsDisplayDateTime(startDate) + } + else { + name += " - " + DateUtil.formatAsDisplayDateTime(surveyId.eventTime) + } + name + } + /** * Rearrange survey data to match the data model. * e.g. [a: [b: [c: 1, d: 2], d: 1], b: [c: 1, d: 2]] => [b: [c: 1, d: 2, a: [d: 1]]] @@ -555,12 +578,13 @@ class ParatooService { output } - private String createSiteFromSurveyData(Map observation, ParatooCollection collection, ParatooCollectionId surveyId, Project project, ParatooProtocolConfig config, ActivityForm form) { + private Map createSiteFromSurveyData(Map observation, ParatooCollection collection, ParatooCollectionId surveyId, Project project, ParatooProtocolConfig config, ActivityForm form) { String siteId = null // Create a site representing the location of the collection + Map siteProps = null Map geoJson = config.getGeoJson(observation, form) if (geoJson) { - Map siteProps = siteService.propertiesFromGeoJson(geoJson, 'upload') + siteProps = siteService.propertiesFromGeoJson(geoJson, 'upload') List features = geoJson?.features ?: [] geoJson.remove('features') siteProps.features = features @@ -592,7 +616,7 @@ class ParatooService { } siteId = result.siteId } - siteId + [siteId:siteId, name:siteProps?.name] } private Map syncParatooProtocols(List protocols) { @@ -774,10 +798,6 @@ class ParatooService { dataSet } - private static String buildSurveyQueryString(int start, int limit, String createdAt) { - "?populate=deep&sort=updatedAt&pagination[start]=$start&pagination[limit]=$limit&filters[createdAt][\$eq]=$createdAt" - } - Map retrieveSurveyAndObservations(ParatooCollection collection, Map authHeader = null) { String apiEndpoint = PARATOO_DATA_PATH Map payload = [ diff --git a/src/main/groovy/au/org/ala/ecodata/DateUtil.groovy b/src/main/groovy/au/org/ala/ecodata/DateUtil.groovy index 8c5831423..ee5d8a0ae 100644 --- a/src/main/groovy/au/org/ala/ecodata/DateUtil.groovy +++ b/src/main/groovy/au/org/ala/ecodata/DateUtil.groovy @@ -54,6 +54,11 @@ class DateUtil { dateTime.format(DISPLAY_DATE_TIME_FORMATTER) } + static String formatAsDisplayDateTime(String isoDateString) { + Date date = parse(isoDateString) + formatAsDisplayDateTime(date) + } + /** * Returns a formatted string representing the financial year a report or activity falls into, based on * the end date. This method won't necessarily work for start dates as it will subtract a day from the value diff --git a/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy b/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy index 180c1350f..5a4b2d038 100644 --- a/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy +++ b/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy @@ -182,7 +182,7 @@ class ParatooServiceSpec extends MongoSpec implements ServiceUnitTest Date: Tue, 14 May 2024 13:28:21 +1000 Subject: [PATCH 16/44] Fixed locale dependency, remove project name from data set name #942 --- .../au/org/ala/ecodata/ParatooService.groovy | 2 +- .../org/ala/ecodata/ParatooServiceSpec.groovy | 20 +++++++++++++------ 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/grails-app/services/au/org/ala/ecodata/ParatooService.groovy b/grails-app/services/au/org/ala/ecodata/ParatooService.groovy index 7a0bd4ef3..ca8709715 100644 --- a/grails-app/services/au/org/ala/ecodata/ParatooService.groovy +++ b/grails-app/services/au/org/ala/ecodata/ParatooService.groovy @@ -196,7 +196,7 @@ class ParatooService { private static String buildName(String protocolId, String displayDate, Project project) { ActivityForm protocolForm = ActivityForm.findByExternalId(protocolId) - String dataSetName = protocolForm?.name + " - " + displayDate + " (" + project.name + ")" + String dataSetName = protocolForm?.name + " - " + displayDate dataSetName } diff --git a/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy b/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy index 5a4b2d038..d6a5351b3 100644 --- a/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy +++ b/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy @@ -12,6 +12,9 @@ import org.codehaus.jackson.map.ObjectMapper import org.grails.web.converters.marshaller.json.CollectionMarshaller import org.grails.web.converters.marshaller.json.MapMarshaller +import java.time.format.DateTimeTextProvider +import java.time.temporal.TemporalField + import static grails.async.Promises.waitAll /** * Tests for the ParatooService. @@ -33,6 +36,10 @@ class ParatooServiceSpec extends MongoSpec implements ServiceUnitTest Date: Wed, 15 May 2024 11:10:40 +1000 Subject: [PATCH 17/44] #941 - added fauna plot to site - site with features are now of compound type - plot update will now create new site --- .../domain/au/org/ala/ecodata/Site.groovy | 10 +++ .../au/org/ala/ecodata/ParatooService.groovy | 22 +++++-- .../paratoo/ParatooProtocolConfig.groovy | 62 +++++++++++++------ 3 files changed, 70 insertions(+), 24 deletions(-) diff --git a/grails-app/domain/au/org/ala/ecodata/Site.groovy b/grails-app/domain/au/org/ala/ecodata/Site.groovy index 8b08eea50..a8fad1aec 100644 --- a/grails-app/domain/au/org/ala/ecodata/Site.groovy +++ b/grails-app/domain/au/org/ala/ecodata/Site.groovy @@ -181,4 +181,14 @@ class Site { status != Status.DELETED }.find() } + + static List findAllByExternalId(ExternalId.IdType idType, String externalId, Map params) { + where { + externalIds { + idType == idType + externalId == externalId + } + status != Status.DELETED + }.list(params) + } } diff --git a/grails-app/services/au/org/ala/ecodata/ParatooService.groovy b/grails-app/services/au/org/ala/ecodata/ParatooService.groovy index 7a0bd4ef3..ee3e5de5f 100644 --- a/grails-app/services/au/org/ala/ecodata/ParatooService.groovy +++ b/grails-app/services/au/org/ala/ecodata/ParatooService.groovy @@ -588,7 +588,10 @@ class ParatooService { List features = geoJson?.features ?: [] geoJson.remove('features') siteProps.features = features - siteProps.type = Site.TYPE_SURVEY_AREA + if (features) + siteProps.type = Site.TYPE_COMPOUND + else + siteProps.type = Site.TYPE_SURVEY_AREA siteProps.publicationStatus = PublicationStatus.PUBLISHED siteProps.projects = [project.projectId] String externalId = geoJson.properties?.externalId @@ -598,18 +601,19 @@ class ParatooService { Site site // create new site for every non-plot submission if (config.usesPlotLayout) { - site = Site.findByExternalId(ExternalId.IdType.MONITOR_PLOT_GUID, externalId) - if (site?.features) { - siteProps.features?.addAll(site.features) - } + List sites = Site.findAllByExternalId(ExternalId.IdType.MONITOR_PLOT_GUID, externalId, [sort: "lastUpdated", order: "desc"]) + if (sites) + site = sites.first() } Map result - if (!site) { + // If the plot layout has been updated, create a new site + if (!site || isUpdatedPlotLayout(site, observation, config)) { result = siteService.create(siteProps) } else { result = [siteId: site.siteId] } + if (result.error) { // Don't treat this as a fatal error for the purposes of responding to the paratoo request log.error("Error creating a site for survey " + collection.orgMintedUUID + ", project " + project.projectId + ": " + result.error) @@ -619,6 +623,12 @@ class ParatooService { [siteId:siteId, name:siteProps?.name] } + private static boolean isUpdatedPlotLayout (Site site, Map observation, ParatooProtocolConfig config) { + Date localSiteUpdated = site.lastUpdated + Date plotLayoutUpdated = config.getPlotLayoutUpdatedAt(observation) + plotLayoutUpdated?.after(localSiteUpdated) ?: false + } + private Map syncParatooProtocols(List protocols) { Map result = [errors: [], messages: []] List guids = [] diff --git a/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy b/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy index 9ccd795c1..83ba04587 100644 --- a/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy +++ b/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy @@ -12,7 +12,8 @@ import org.locationtech.jts.geom.Geometry @Slf4j @JsonIgnoreProperties(['metaClass', 'errors', 'expandoMetaClass']) class ParatooProtocolConfig { - + static final String FAUNA_PLOT = 'Fauna plot' + static final String CORE_PLOT = 'Core monitoring plot' String name String apiEndpoint boolean usesPlotLayout = true @@ -26,7 +27,9 @@ class ParatooProtocolConfig { String plotVisitPath = 'plot_visit' String plotLayoutPath = "${plotVisitPath}.plot_layout" String plotLayoutIdPath = "${plotLayoutPath}.id" + String plotLayoutUpdatedAtPath = "${plotLayoutPath}.updatedAt" String plotLayoutPointsPath = "${plotLayoutPath}.plot_points" + String faunaPlotPointPath = "${plotLayoutPath}.fauna_plot_point" String plotSelectionPath = "${plotLayoutPath}.plot_selection" String plotLayoutDimensionLabelPath = "${plotLayoutPath}.plot_dimensions.label" String plotLayoutTypeLabelPath = "${plotLayoutPath}.plot_type.label" @@ -79,6 +82,16 @@ class ParatooProtocolConfig { return removeMilliseconds(date) } + Date getPlotLayoutUpdatedAt(Map surveyData) { + def date = getProperty(surveyData, plotLayoutUpdatedAtPath) + if (!date) { + date = getPropertyFromSurvey(surveyData, plotLayoutUpdatedAtPath) + } + + date = getFirst(date) + date ? DateUtil.parseWithMilliseconds(date) : null + } + Map getSurveyId(Map surveyData) { if(surveyIdPath == null || surveyData == null) { return null @@ -203,7 +216,11 @@ class ParatooProtocolConfig { geoJson = extractSiteDataFromPlotVisit(output) // get list of all features associated with observation if (geoJson && form && output) { - geoJson.features = extractFeatures(output, form) + List features = extractFeatures(output, form) + if (features) { + features.addAll(geoJson.features?:[]) + geoJson = createConvexHullGeoJSON(features, geoJson.properties.name, geoJson.properties.externalId) + } } } else if (geometryPath) { @@ -212,20 +229,10 @@ class ParatooProtocolConfig { else if (form && output) { List features = extractFeatures(output, form) if (features) { - List featureGeometries = features.collect { it.geometry } - Geometry geometry = GeometryUtils.getFeatureCollectionConvexHull(featureGeometries) String startDateInString = getStartDate(output) startDateInString = DateUtil.convertUTCDateToStringInTimeZone(startDateInString, clientTimeZone?:TimeZone.default) String name = "${form.name} site - ${startDateInString}" - geoJson = [ - type: 'Feature', - geometry: GeometryUtils.geometryToGeoJsonMap(geometry), - properties: [ - name: name, - description: "${name} (convex hull of all features)", - ], - features: features - ] + geoJson = createConvexHullGeoJSON(features, name) } } @@ -313,6 +320,8 @@ class ParatooProtocolConfig { log.warn("No plot_layout found in survey at path ${plotLayoutIdPath}") return null } + else + plotLayoutId = plotLayoutId.toString() List plotLayoutPoints = getProperty(surveyData, plotLayoutPointsPath) Map plotSelection = getProperty(surveyData, plotSelectionPath) Map plotSelectionGeoJson = plotSelectionToGeoJson(plotSelection) @@ -322,16 +331,33 @@ class ParatooProtocolConfig { String name = plotSelectionGeoJson.properties.name + ' - ' + plotLayoutTypeLabel + ' (' + plotLayoutDimensionLabel + ')' - Map plotGeoJson = createFeatureFromGeoJSON(plotLayoutPoints, name, plotLayoutId, plotSelectionGeoJson?.properties?.notes) - - //Map faunaPlotGeoJson = toGeometry(plotLayout.fauna_plot_point) + Map plotGeoJson = createFeatureFromGeoJSON(plotLayoutPoints, name, plotLayoutId, "${CORE_PLOT} ${plotSelectionGeoJson?.properties?.notes?:""}") + List faunaPlotPoints = getProperty(surveyData, faunaPlotPointPath) + Map faunaPlotGeoJson = createFeatureFromGeoJSON(faunaPlotPoints, name, plotLayoutId, "${FAUNA_PLOT} ${plotSelectionGeoJson?.properties?.notes?:""}") - // TODO maybe turn this into a feature with properties to distinguish the fauna plot? - // Or a multi-polygon? + if (faunaPlotGeoJson) { + List features = [plotGeoJson, faunaPlotGeoJson] + plotGeoJson = createConvexHullGeoJSON(features, name, plotLayoutId) + } plotGeoJson } + static Map createConvexHullGeoJSON (List features, String name, String externalId = "") { + List featureGeometries = features.collect { it.geometry } + Geometry geometry = GeometryUtils.getFeatureCollectionConvexHull(featureGeometries) + [ + type: 'Feature', + geometry: GeometryUtils.geometryToGeoJsonMap(geometry), + properties: [ + name: name, + externalId: externalId, + description: "${name} (convex hull of all features)", + ], + features: features + ] + } + static Map createFeatureFromGeoJSON(List plotLayoutPoints, String name, def plotLayoutId, String notes = "") { Map plotGeometry = toGeometry(plotLayoutPoints) createFeatureObject(plotGeometry, name, plotLayoutId, notes) From dccc4f405ab262efeec0899dcee4f964f2abff95 Mon Sep 17 00:00:00 2001 From: temi Date: Wed, 15 May 2024 12:37:13 +1000 Subject: [PATCH 18/44] #941 - external ids are now of type string - fixed broken tests --- .../paratoo/ParatooProtocolConfig.groovy | 22 ++--- .../org/ala/ecodata/ParatooServiceSpec.groovy | 6 +- .../paratoo/ParatooProtocolConfigSpec.groovy | 93 +++++++++++++++---- 3 files changed, 91 insertions(+), 30 deletions(-) diff --git a/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy b/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy index 83ba04587..c3187dcd4 100644 --- a/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy +++ b/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy @@ -219,7 +219,7 @@ class ParatooProtocolConfig { List features = extractFeatures(output, form) if (features) { features.addAll(geoJson.features?:[]) - geoJson = createConvexHullGeoJSON(features, geoJson.properties.name, geoJson.properties.externalId) + geoJson = createConvexHullGeoJSON(features, geoJson.properties.name, geoJson.properties.externalId, geoJson.properties.notes) } } } @@ -314,14 +314,13 @@ class ParatooProtocolConfig { private Map extractSiteDataFromPlotVisit(Map survey) { Map surveyData = getSurveyData(survey) - def plotLayoutId = getProperty(surveyData, plotLayoutIdPath) // Currently an int, may become uuid? + String plotLayoutId = getProperty(surveyData, plotLayoutIdPath) // Currently an int, may become uuid? if (!plotLayoutId) { log.warn("No plot_layout found in survey at path ${plotLayoutIdPath}") return null } - else - plotLayoutId = plotLayoutId.toString() + List plotLayoutPoints = getProperty(surveyData, plotLayoutPointsPath) Map plotSelection = getProperty(surveyData, plotSelectionPath) Map plotSelectionGeoJson = plotSelectionToGeoJson(plotSelection) @@ -333,17 +332,17 @@ class ParatooProtocolConfig { Map plotGeoJson = createFeatureFromGeoJSON(plotLayoutPoints, name, plotLayoutId, "${CORE_PLOT} ${plotSelectionGeoJson?.properties?.notes?:""}") List faunaPlotPoints = getProperty(surveyData, faunaPlotPointPath) - Map faunaPlotGeoJson = createFeatureFromGeoJSON(faunaPlotPoints, name, plotLayoutId, "${FAUNA_PLOT} ${plotSelectionGeoJson?.properties?.notes?:""}") - if (faunaPlotGeoJson) { + if (faunaPlotPoints) { + Map faunaPlotGeoJson = createFeatureFromGeoJSON(faunaPlotPoints, name, plotLayoutId, "${FAUNA_PLOT} ${plotSelectionGeoJson?.properties?.notes?:""}") List features = [plotGeoJson, faunaPlotGeoJson] - plotGeoJson = createConvexHullGeoJSON(features, name, plotLayoutId) + plotGeoJson = createConvexHullGeoJSON(features, name, plotLayoutId, plotGeoJson.properties.notes) } plotGeoJson } - static Map createConvexHullGeoJSON (List features, String name, String externalId = "") { + static Map createConvexHullGeoJSON (List features, String name, String externalId = "", String notes = "") { List featureGeometries = features.collect { it.geometry } Geometry geometry = GeometryUtils.getFeatureCollectionConvexHull(featureGeometries) [ @@ -352,18 +351,19 @@ class ParatooProtocolConfig { properties: [ name: name, externalId: externalId, + notes: notes, description: "${name} (convex hull of all features)", ], features: features ] } - static Map createFeatureFromGeoJSON(List plotLayoutPoints, String name, def plotLayoutId, String notes = "") { + static Map createFeatureFromGeoJSON(List plotLayoutPoints, String name, String plotLayoutId, String notes = "") { Map plotGeometry = toGeometry(plotLayoutPoints) createFeatureObject(plotGeometry, name, plotLayoutId, notes) } - static Map createFeatureObject(Map plotGeometry, String name, plotLayoutId, String notes = "") { + static Map createFeatureObject(Map plotGeometry, String name, String plotLayoutId, String notes = "") { [ type : 'Feature', geometry : plotGeometry, @@ -388,7 +388,7 @@ class ParatooProtocolConfig { plotGeometry } - static Map createLineStringFeatureFromGeoJSON (List plotLayoutPoints, String name, def plotLayoutId, String notes = "") { + static Map createLineStringFeatureFromGeoJSON (List plotLayoutPoints, String name, String plotLayoutId, String notes = "") { Map plotGeometry = toLineStringGeometry(plotLayoutPoints) createFeatureObject(plotGeometry, name, plotLayoutId, notes) } diff --git a/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy b/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy index 5a4b2d038..3061c7c5e 100644 --- a/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy +++ b/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy @@ -332,9 +332,9 @@ class ParatooServiceSpec extends MongoSpec implements ServiceUnitTest Date: Tue, 14 May 2024 13:28:21 +1000 Subject: [PATCH 19/44] Fixed locale dependency, remove project name from data set name #942 --- .../au/org/ala/ecodata/ParatooService.groovy | 2 +- .../org/ala/ecodata/ParatooServiceSpec.groovy | 20 +++++++++++++------ 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/grails-app/services/au/org/ala/ecodata/ParatooService.groovy b/grails-app/services/au/org/ala/ecodata/ParatooService.groovy index ee3e5de5f..8118981cc 100644 --- a/grails-app/services/au/org/ala/ecodata/ParatooService.groovy +++ b/grails-app/services/au/org/ala/ecodata/ParatooService.groovy @@ -196,7 +196,7 @@ class ParatooService { private static String buildName(String protocolId, String displayDate, Project project) { ActivityForm protocolForm = ActivityForm.findByExternalId(protocolId) - String dataSetName = protocolForm?.name + " - " + displayDate + " (" + project.name + ")" + String dataSetName = protocolForm?.name + " - " + displayDate dataSetName } diff --git a/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy b/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy index 3061c7c5e..47aebe431 100644 --- a/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy +++ b/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy @@ -12,6 +12,9 @@ import org.codehaus.jackson.map.ObjectMapper import org.grails.web.converters.marshaller.json.CollectionMarshaller import org.grails.web.converters.marshaller.json.MapMarshaller +import java.time.format.DateTimeTextProvider +import java.time.temporal.TemporalField + import static grails.async.Promises.waitAll /** * Tests for the ParatooService. @@ -33,6 +36,10 @@ class ParatooServiceSpec extends MongoSpec implements ServiceUnitTest Date: Wed, 15 May 2024 13:51:07 +1000 Subject: [PATCH 20/44] #941 - removes adding observation features to plot site --- .../ala/ecodata/paratoo/ParatooProtocolConfig.groovy | 8 -------- .../ecodata/paratoo/ParatooProtocolConfigSpec.groovy | 12 ------------ 2 files changed, 20 deletions(-) diff --git a/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy b/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy index c3187dcd4..5fe58fd2f 100644 --- a/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy +++ b/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy @@ -214,14 +214,6 @@ class ParatooProtocolConfig { Map geoJson = null if (usesPlotLayout) { geoJson = extractSiteDataFromPlotVisit(output) - // get list of all features associated with observation - if (geoJson && form && output) { - List features = extractFeatures(output, form) - if (features) { - features.addAll(geoJson.features?:[]) - geoJson = createConvexHullGeoJSON(features, geoJson.properties.name, geoJson.properties.externalId, geoJson.properties.notes) - } - } } else if (geometryPath) { geoJson = extractSiteDataFromPath(output) diff --git a/src/test/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfigSpec.groovy b/src/test/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfigSpec.groovy index 7ab8937f5..651031e85 100644 --- a/src/test/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfigSpec.groovy +++ b/src/test/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfigSpec.groovy @@ -218,18 +218,6 @@ class ParatooProtocolConfigSpec extends Specification { ] ] config.getGeoJson(observation, activityForm).features == [ - [ - type:"Feature", - geometry:[ - type:"Point", - coordinates:[149.0651491, -35.2592444] - ], - properties:[ - name:"Point aParatooForm 2-1", - externalId:37, - id:"aParatooForm 2-1" - ] - ], [ type:"Feature", geometry:[ From e8150775f1c521828a2b24b12fe7cd2cbb39bd89 Mon Sep 17 00:00:00 2001 From: temi Date: Wed, 15 May 2024 16:29:47 +1000 Subject: [PATCH 21/44] #941 - fixes null pointer issue where geometry points are null or an empty list --- .../au/org/ala/ecodata/ParatooService.groovy | 58 ++++++++++--------- .../paratoo/ParatooProtocolConfig.groovy | 4 +- 2 files changed, 34 insertions(+), 28 deletions(-) diff --git a/grails-app/services/au/org/ala/ecodata/ParatooService.groovy b/grails-app/services/au/org/ala/ecodata/ParatooService.groovy index 8118981cc..687d96936 100644 --- a/grails-app/services/au/org/ala/ecodata/ParatooService.groovy +++ b/grails-app/services/au/org/ala/ecodata/ParatooService.groovy @@ -535,35 +535,39 @@ class ParatooService { // used by protocols like bird survey where a point represents a sight a bird has been observed in a // bird survey plot def location = output[model.name] - if (location instanceof Map) { - output[model.name] = [ - type : 'Feature', - geometry : [ - type : 'Point', - coordinates: [location.lng, location.lat] - ], - properties: [ - name : "Point ${formName}-${featureId}", - externalId: location.id, - id: "${formName}-${featureId}" - ] - ] - } - else if (location instanceof List) { - String name - switch (config?.geometryType) { - case "LineString": - name = "LineString ${formName}-${featureId}" - output[model.name] = ParatooProtocolConfig.createLineStringFeatureFromGeoJSON (location, name, null, name) - break - default: - name = "Polygon ${formName}-${featureId}" - output[model.name] = ParatooProtocolConfig.createFeatureFromGeoJSON (location, name, null, name) - break + if (location) { + if (location instanceof Map) { + output[model.name] = [ + type : 'Feature', + geometry : [ + type : 'Point', + coordinates: [location.lng, location.lat] + ], + properties: [ + name : "Point ${formName}-${featureId}", + externalId: location.id, + id : "${formName}-${featureId}" + ] + ] + } else if (location instanceof List) { + String name + switch (config?.geometryType) { + case "LineString": + name = "LineString ${formName}-${featureId}" + output[model.name] = ParatooProtocolConfig.createLineStringFeatureFromGeoJSON(location, name, null, name) + break + default: + name = "Polygon ${formName}-${featureId}" + output[model.name] = ParatooProtocolConfig.createFeatureFromGeoJSON(location, name, null, name) + break + } } - } - featureId ++ + featureId ++ + } + else { + output[model.name] = null + } break case "image": case "document": diff --git a/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy b/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy index 5fe58fd2f..68bc3277e 100644 --- a/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy +++ b/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy @@ -148,7 +148,8 @@ class ParatooProtocolConfig { List features = [] paths.each { String name, node -> if (node instanceof Boolean) { - features.add(output[name]) + if (output[name]) + features.add(output[name]) // todo later: add featureIds and modelId for compliance with feature behaviour of reports } @@ -335,6 +336,7 @@ class ParatooProtocolConfig { } static Map createConvexHullGeoJSON (List features, String name, String externalId = "", String notes = "") { + features = features.findAll { it.geometry != null } List featureGeometries = features.collect { it.geometry } Geometry geometry = GeometryUtils.getFeatureCollectionConvexHull(featureGeometries) [ From 383462af870ebdec43ac87b2476cf5c0ef5eb56d Mon Sep 17 00:00:00 2001 From: chrisala Date: Thu, 16 May 2024 08:42:53 +1000 Subject: [PATCH 22/44] Optionally adds species URL to download #945 --- grails-app/conf/application.groovy | 3 + .../ecodata/metadata/SpeciesUrlGetter.groovy | 23 +++++++ .../reporting/ProjectXlsExporter.groovy | 2 + .../ecodata/reporting/TabbedExporter.groovy | 19 ++++++ .../reporting/TabbedExporterSpec.groovy | 66 +++++++++++++++++++ 5 files changed, 113 insertions(+) create mode 100644 src/main/groovy/au/org/ala/ecodata/metadata/SpeciesUrlGetter.groovy diff --git a/grails-app/conf/application.groovy b/grails-app/conf/application.groovy index 3a77d5315..2938f86e0 100644 --- a/grails-app/conf/application.groovy +++ b/grails-app/conf/application.groovy @@ -501,6 +501,9 @@ app { if (!ala.baseURL) { ala.baseURL = "https://www.ala.org.au" } +bie.ws.url = "https://www.bie.ala.org.au/ws" +bie.url = "https://www.bie.ala.org.au" + if (!collectory.baseURL) { //collectory.baseURL = "https://collectory-dev.ala.org.au/" collectory.baseURL = "https://collections-test.ala.org.au/" diff --git a/src/main/groovy/au/org/ala/ecodata/metadata/SpeciesUrlGetter.groovy b/src/main/groovy/au/org/ala/ecodata/metadata/SpeciesUrlGetter.groovy new file mode 100644 index 000000000..296fe17eb --- /dev/null +++ b/src/main/groovy/au/org/ala/ecodata/metadata/SpeciesUrlGetter.groovy @@ -0,0 +1,23 @@ +package au.org.ala.ecodata.metadata + +import pl.touk.excel.export.getters.Getter + +class SpeciesUrlGetter extends OutputDataGetter implements Getter { + String biePrefix + SpeciesUrlGetter(String propertyName, Map dataNode, Map documentMap, TimeZone timeZone, String biePrefix) { + super(propertyName, dataNode, documentMap, timeZone) + this.biePrefix = biePrefix + } + + @Override + def species(Object node, Value outputValue) { + def val = outputValue.value + if (!val?.name) { + return "" + } + + return val?.guid ? biePrefix+val.guid : "Unmatched name" + } + + +} diff --git a/src/main/groovy/au/org/ala/ecodata/reporting/ProjectXlsExporter.groovy b/src/main/groovy/au/org/ala/ecodata/reporting/ProjectXlsExporter.groovy index ff2e8e1cc..7d236ee70 100644 --- a/src/main/groovy/au/org/ala/ecodata/reporting/ProjectXlsExporter.groovy +++ b/src/main/groovy/au/org/ala/ecodata/reporting/ProjectXlsExporter.groovy @@ -176,6 +176,7 @@ class ProjectXlsExporter extends ProjectExporter { super(exporter) this.projectService = projectService distinctElectorates = new ArrayList() + useSpeciesUrlGetter = true setupManagementUnits(managementUnitService) setupFundingAbn(organisationService) setupProgramData(programService) @@ -185,6 +186,7 @@ class ProjectXlsExporter extends ProjectExporter { super(exporter, tabsToExport, [:], TimeZone.default) this.projectService = projectService this.formSectionPerTab = formSectionPerTab + useSpeciesUrlGetter = true addDataDescriptionToDownload(downloadMetadata) distinctElectorates = new ArrayList(electorates?:[]) distinctElectorates.sort() diff --git a/src/main/groovy/au/org/ala/ecodata/reporting/TabbedExporter.groovy b/src/main/groovy/au/org/ala/ecodata/reporting/TabbedExporter.groovy index 4664a347e..17efdc3f9 100644 --- a/src/main/groovy/au/org/ala/ecodata/reporting/TabbedExporter.groovy +++ b/src/main/groovy/au/org/ala/ecodata/reporting/TabbedExporter.groovy @@ -6,6 +6,7 @@ import au.org.ala.ecodata.metadata.OutputDateGetter import au.org.ala.ecodata.metadata.OutputMetadata import au.org.ala.ecodata.metadata.OutputModelProcessor import au.org.ala.ecodata.metadata.OutputNumberGetter +import au.org.ala.ecodata.metadata.SpeciesUrlGetter import grails.util.Holders import org.apache.commons.logging.Log import org.apache.commons.logging.LogFactory @@ -26,6 +27,7 @@ class TabbedExporter { ReportingService reportingService = Holders.grailsApplication.mainContext.getBean("reportingService") ActivityFormService activityFormService = Holders.grailsApplication.mainContext.getBean("activityFormService") OutputModelProcessor processor = new OutputModelProcessor() + String biePrefix = Holders.grailsApplication.config.getProperty("bie.url")+'/species/' static String DATE_CELL_FORMAT = "dd/MM/yyyy" Map sheets @@ -35,6 +37,7 @@ class TabbedExporter { TimeZone timeZone Boolean useDateGetter = false Boolean useNumberGetter = false + boolean useSpeciesUrlGetter = false // These fields map full activity names to shortened names that are compatible with Excel tabs. protected Map activitySheetNames = [:] protected Map> typedActivitySheets = [:] @@ -232,6 +235,22 @@ class TabbedExporter { getter:new OutputNumberGetter(propertyPath, dataNode, documentMap, timeZone)] fieldConfiguration << field } + else if ((dataNode.dataType == 'species') && useSpeciesUrlGetter) { + // Return a property for the species name and a property for the species URL + Map nameField = field + [ + header:outputMetadata.getLabel(viewNode, dataNode), + property:propertyPath, + getter:new OutputDataGetter(propertyPath, dataNode, documentMap, timeZone)] + fieldConfiguration << nameField + + Map urlField = field + [ + description: "Link to species in the ALA", + header:outputMetadata.getLabel(viewNode, dataNode), + property:propertyPath, + getter:new SpeciesUrlGetter(propertyPath, dataNode, documentMap, timeZone, biePrefix) + ] + fieldConfiguration << urlField + } else { field += [ header:outputMetadata.getLabel(viewNode, dataNode), diff --git a/src/test/groovy/au/org/ala/ecodata/reporting/TabbedExporterSpec.groovy b/src/test/groovy/au/org/ala/ecodata/reporting/TabbedExporterSpec.groovy index 81d261aa5..695749d37 100644 --- a/src/test/groovy/au/org/ala/ecodata/reporting/TabbedExporterSpec.groovy +++ b/src/test/groovy/au/org/ala/ecodata/reporting/TabbedExporterSpec.groovy @@ -1,6 +1,8 @@ package au.org.ala.ecodata.reporting import au.org.ala.ecodata.* +import au.org.ala.ecodata.metadata.OutputDataGetter +import au.org.ala.ecodata.metadata.SpeciesUrlGetter import grails.testing.gorm.DomainUnitTest import grails.testing.web.GrailsWebUnitTest import grails.util.Holders @@ -97,6 +99,49 @@ class TabbedExporterSpec extends Specification implements GrailsWebUnitTest, Dom } + def "Species data types will be expanded into two export columns if useSpeciesUrlGetter is true"() { + setup: + String type = 'form' + ActivityForm form = buildMockForm(type, buildFormTemplateWithSpecies()) + + when: + tabbedExporter.useSpeciesUrlGetter = true + List config = tabbedExporter.getActivityExportConfig(type, true) + + then: + 1 * activityFormService.findVersionedActivityForm(type) >> [form] + + config.size() == 3 + config[1].header == 'Species label' + config[1].property == 'form.species' + config[1].getter instanceof OutputDataGetter + + config[2].header == 'Species label' + config[2].property == 'form.species' + config[2].getter instanceof SpeciesUrlGetter + + } + + def "Species data types will only export the species name if useSpeciesUrlGetter is false"() { + setup: + String type = 'form' + ActivityForm form = buildMockForm(type, buildFormTemplateWithSpecies()) + + when: + tabbedExporter.useSpeciesUrlGetter = false + List config = tabbedExporter.getActivityExportConfig(type, true) + + then: + 1 * activityFormService.findVersionedActivityForm(type) >> [form] + + config.size() == 2 + config[1].header == 'Species label' + config[1].property == 'form.species' + config[1].getter instanceof OutputDataGetter + } + + + private ActivityForm buildMockForm(String name, Map template) { ActivityForm form = new ActivityForm(name:name, formVersion:1) FormSection section = new FormSection(name:name, template:template) @@ -142,4 +187,25 @@ class TabbedExporterSpec extends Specification implements GrailsWebUnitTest, Dom ] ] } + + private Map buildFormTemplateWithSpecies() { + [ + dataModel:[ + [ + name:"species", + dataType:"species" + ] + ], + viewModel:[ + [type:'row', items:[ + [ + type:'speciesSelect', + source:'species', + preLabel:'Species label' + ] + ]] + + ] + ] + } } From 1d63d99a077421c26ef41b6074c72a2ff31a112b Mon Sep 17 00:00:00 2001 From: chrisala Date: Thu, 16 May 2024 11:01:47 +1000 Subject: [PATCH 23/44] Addressed code review comments #945 --- grails-app/conf/application.groovy | 4 ++-- grails-app/domain/au/org/ala/ecodata/Record.groovy | 2 ++ .../services/au/org/ala/ecodata/ParatooService.groovy | 2 +- .../au/org/ala/ecodata/SpeciesReMatchService.groovy | 2 +- .../au/org/ala/ecodata/metadata/SpeciesUrlGetter.groovy | 7 +++++-- .../au/org/ala/ecodata/reporting/TabbedExporter.groovy | 2 +- 6 files changed, 12 insertions(+), 7 deletions(-) diff --git a/grails-app/conf/application.groovy b/grails-app/conf/application.groovy index 2938f86e0..3f0209f77 100644 --- a/grails-app/conf/application.groovy +++ b/grails-app/conf/application.groovy @@ -501,8 +501,8 @@ app { if (!ala.baseURL) { ala.baseURL = "https://www.ala.org.au" } -bie.ws.url = "https://www.bie.ala.org.au/ws" -bie.url = "https://www.bie.ala.org.au" +bie.ws.url = "https://bie-ws.ala.org.au/" +bie.url = "https://bie.ala.org.au/" if (!collectory.baseURL) { //collectory.baseURL = "https://collectory-dev.ala.org.au/" diff --git a/grails-app/domain/au/org/ala/ecodata/Record.groovy b/grails-app/domain/au/org/ala/ecodata/Record.groovy index 0dfe00481..ddfd8682b 100644 --- a/grails-app/domain/au/org/ala/ecodata/Record.groovy +++ b/grails-app/domain/au/org/ala/ecodata/Record.groovy @@ -6,6 +6,8 @@ import org.bson.types.ObjectId class Record { // def grailsApplication + /** Represents a species guid that was unable to be matched against the ALA names list */ + static final String UNMATCHED_GUID = "A_GUID" static mapping = { occurrenceID index: true diff --git a/grails-app/services/au/org/ala/ecodata/ParatooService.groovy b/grails-app/services/au/org/ala/ecodata/ParatooService.groovy index ca8709715..7e909bb6a 100644 --- a/grails-app/services/au/org/ala/ecodata/ParatooService.groovy +++ b/grails-app/services/au/org/ala/ecodata/ParatooService.groovy @@ -2052,7 +2052,7 @@ class ParatooService { } // record is only created if guid is present - result.guid = result.guid ?: "A_GUID" + result.guid = result.guid ?: Record.UNMATCHED_GUID result } } diff --git a/grails-app/services/au/org/ala/ecodata/SpeciesReMatchService.groovy b/grails-app/services/au/org/ala/ecodata/SpeciesReMatchService.groovy index e074cd730..11214a11d 100644 --- a/grails-app/services/au/org/ala/ecodata/SpeciesReMatchService.groovy +++ b/grails-app/services/au/org/ala/ecodata/SpeciesReMatchService.groovy @@ -85,7 +85,7 @@ class SpeciesReMatchService { name = name?.toLowerCase() ?: "" cacheService.get('bie-search-auto-' + name, { def encodedQuery = URLEncoder.encode(name ?: '', "UTF-8") - def url = "${grailsApplication.config.getProperty('bie.url')}ws/search/auto.jsonp?q=${encodedQuery}&limit=${limit}&idxType=TAXON" + def url = "${grailsApplication.config.getProperty('bie.ws.url')}ws/search/auto.jsonp?q=${encodedQuery}&limit=${limit}&idxType=TAXON" webService.getJson(url) }) diff --git a/src/main/groovy/au/org/ala/ecodata/metadata/SpeciesUrlGetter.groovy b/src/main/groovy/au/org/ala/ecodata/metadata/SpeciesUrlGetter.groovy index 296fe17eb..89a7dffc1 100644 --- a/src/main/groovy/au/org/ala/ecodata/metadata/SpeciesUrlGetter.groovy +++ b/src/main/groovy/au/org/ala/ecodata/metadata/SpeciesUrlGetter.groovy @@ -1,6 +1,7 @@ package au.org.ala.ecodata.metadata import pl.touk.excel.export.getters.Getter +import au.org.ala.ecodata.Record class SpeciesUrlGetter extends OutputDataGetter implements Getter { String biePrefix @@ -15,8 +16,10 @@ class SpeciesUrlGetter extends OutputDataGetter implements Getter { if (!val?.name) { return "" } - - return val?.guid ? biePrefix+val.guid : "Unmatched name" + if (!val?.guid || val.guid == Record.UNMATCHED_GUID) { + return "Unmatched name" + } + return biePrefix+val.guid } diff --git a/src/main/groovy/au/org/ala/ecodata/reporting/TabbedExporter.groovy b/src/main/groovy/au/org/ala/ecodata/reporting/TabbedExporter.groovy index 17efdc3f9..3151e2663 100644 --- a/src/main/groovy/au/org/ala/ecodata/reporting/TabbedExporter.groovy +++ b/src/main/groovy/au/org/ala/ecodata/reporting/TabbedExporter.groovy @@ -27,7 +27,7 @@ class TabbedExporter { ReportingService reportingService = Holders.grailsApplication.mainContext.getBean("reportingService") ActivityFormService activityFormService = Holders.grailsApplication.mainContext.getBean("activityFormService") OutputModelProcessor processor = new OutputModelProcessor() - String biePrefix = Holders.grailsApplication.config.getProperty("bie.url")+'/species/' + String biePrefix = Holders.grailsApplication.config.getProperty("bie.url")+'species/' static String DATE_CELL_FORMAT = "dd/MM/yyyy" Map sheets From 84d425a19ba2dad91074181851ccf02315457bfb Mon Sep 17 00:00:00 2001 From: chrisala Date: Thu, 16 May 2024 14:06:24 +1000 Subject: [PATCH 24/44] size -> size() #947 --- .../groovy/au/org/ala/ecodata/reporting/PropertyAccessor.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/groovy/au/org/ala/ecodata/reporting/PropertyAccessor.groovy b/src/main/groovy/au/org/ala/ecodata/reporting/PropertyAccessor.groovy index 178a34e03..17b0ca042 100644 --- a/src/main/groovy/au/org/ala/ecodata/reporting/PropertyAccessor.groovy +++ b/src/main/groovy/au/org/ala/ecodata/reporting/PropertyAccessor.groovy @@ -84,7 +84,7 @@ class PropertyAccessor { results.each { item -> tmpResults.addAll(unrollSingle(item)) } - if (tmpResults.size > 0) { + if (tmpResults.size() > 0) { results = tmpResults } } From 93ad03de8331da6d971c6a3066df0d1b213385a4 Mon Sep 17 00:00:00 2001 From: temi Date: Thu, 16 May 2024 14:08:47 +1000 Subject: [PATCH 25/44] #941 - fixes NPE when calculation convex hull - new site for edited site --- .../au/org/ala/ecodata/AdminController.groovy | 2 +- .../au/org/ala/ecodata/ParatooService.groovy | 39 ++++-- .../ecodata/converter/FeatureConverter.groovy | 30 ++--- .../paratoo/ParatooProtocolConfig.groovy | 4 +- .../org/ala/ecodata/ParatooServiceSpec.groovy | 121 +++++++++++++++++- .../au/org/ala/ecodata/SiteServiceSpec.groovy | 23 +++- .../paratoo/ParatooProtocolConfigSpec.groovy | 81 +++++++++++- 7 files changed, 264 insertions(+), 36 deletions(-) diff --git a/grails-app/controllers/au/org/ala/ecodata/AdminController.groovy b/grails-app/controllers/au/org/ala/ecodata/AdminController.groovy index 82fcd699a..a02d4aac7 100644 --- a/grails-app/controllers/au/org/ala/ecodata/AdminController.groovy +++ b/grails-app/controllers/au/org/ala/ecodata/AdminController.groovy @@ -737,7 +737,7 @@ class AdminController { def reSubmitDataSet() { String projectId = params.id String dataSetId = params.dataSetId - String userId = params.userId ?: userService.getCurrentUser().userId + String userId = params.userId ?: userService.currentUser()?.userId if (!projectId || !dataSetId || !userId) { render text: [message: "Bad request"] as JSON, status: HttpStatus.SC_BAD_REQUEST return diff --git a/grails-app/services/au/org/ala/ecodata/ParatooService.groovy b/grails-app/services/au/org/ala/ecodata/ParatooService.groovy index 687d96936..4ddc312d9 100644 --- a/grails-app/services/au/org/ala/ecodata/ParatooService.groovy +++ b/grails-app/services/au/org/ala/ecodata/ParatooService.groovy @@ -279,9 +279,14 @@ class ParatooService { String siteName = null if (!dataSet.siteId) { - Map site = createSiteFromSurveyData(surveyDataAndObservations, collection, surveyId, project.project, config, form) - dataSet.siteId = site.siteId - siteName = site.name + try { + Map site = createSiteFromSurveyData(surveyDataAndObservations, collection, surveyId, project.project, config, form) + dataSet.siteId = site.siteId + siteName = site.name + } + catch (Exception ex) { + log.error("Error creating site for ${collection.orgMintedUUID}: ${ex.message}") + } } // plot layout is of type geoMap. Therefore, expects a site id. @@ -584,6 +589,7 @@ class ParatooService { private Map createSiteFromSurveyData(Map observation, ParatooCollection collection, ParatooCollectionId surveyId, Project project, ParatooProtocolConfig config, ActivityForm form) { String siteId = null + Date updatedPlotLayoutDate // Create a site representing the location of the collection Map siteProps = null Map geoJson = config.getGeoJson(observation, form) @@ -605,6 +611,7 @@ class ParatooService { Site site // create new site for every non-plot submission if (config.usesPlotLayout) { + updatedPlotLayoutDate = config.getPlotLayoutUpdatedAt(observation) List sites = Site.findAllByExternalId(ExternalId.IdType.MONITOR_PLOT_GUID, externalId, [sort: "lastUpdated", order: "desc"]) if (sites) site = sites.first() @@ -612,9 +619,14 @@ class ParatooService { Map result // If the plot layout has been updated, create a new site - if (!site || isUpdatedPlotLayout(site, observation, config)) { + if (!site) { result = siteService.create(siteProps) - } else { + } + else if(isUpdatedPlotLayout(site.lastUpdated, updatedPlotLayoutDate)){ + siteProps.name = "${siteProps.name} - ${DateUtil.formatAsDisplayDateTime(updatedPlotLayoutDate)}" + result = siteService.create(siteProps) + } + else { result = [siteId: site.siteId] } @@ -627,10 +639,19 @@ class ParatooService { [siteId:siteId, name:siteProps?.name] } - private static boolean isUpdatedPlotLayout (Site site, Map observation, ParatooProtocolConfig config) { - Date localSiteUpdated = site.lastUpdated - Date plotLayoutUpdated = config.getPlotLayoutUpdatedAt(observation) - plotLayoutUpdated?.after(localSiteUpdated) ?: false + /** + * check if the plot layout has been updated after site has been updated. This means user has edited plot layout and + * a new site should be created. + * @param siteLastUpdated + * @param plotLayoutLastUpdated + * @return + */ + static boolean isUpdatedPlotLayout (Date siteLastUpdated, Date plotLayoutLastUpdated) { + if ((siteLastUpdated != null) && (plotLayoutLastUpdated != null)) { + return plotLayoutLastUpdated.after(siteLastUpdated) + } + + return false } private Map syncParatooProtocols(List protocols) { diff --git a/src/main/groovy/au/org/ala/ecodata/converter/FeatureConverter.groovy b/src/main/groovy/au/org/ala/ecodata/converter/FeatureConverter.groovy index 03165ef20..3a4cae88a 100644 --- a/src/main/groovy/au/org/ala/ecodata/converter/FeatureConverter.groovy +++ b/src/main/groovy/au/org/ala/ecodata/converter/FeatureConverter.groovy @@ -4,27 +4,27 @@ class FeatureConverter implements RecordFieldConverter { List convert(Map data, Map metadata = [:]) { Map record = [:] + if (data[metadata.name]) { + Double latitude = getDecimalLatitude(data[metadata.name]) + Double longitude = getDecimalLongitude(data[metadata.name]) + // Don't override decimalLongitud or decimalLatitude in case they are null, site info could've already set them + if (latitude != null) { + record.decimalLatitude = latitude + } - Double latitude = getDecimalLatitude(data[metadata.name]) - Double longitude = getDecimalLongitude(data[metadata.name]) - - // Don't override decimalLongitud or decimalLatitude in case they are null, site info could've already set them - if(latitude != null) { - record.decimalLatitude = latitude - } - - if (longitude != null) { - record.decimalLongitude = longitude - } + if (longitude != null) { + record.decimalLongitude = longitude + } - Map dwcMappings = extractDwcMapping(metadata) + Map dwcMappings = extractDwcMapping(metadata) - record << getDwcAttributes(data, dwcMappings) + record << getDwcAttributes(data, dwcMappings) - if (data.dwcAttribute) { - record[data.dwcAttribute] = data.value + if (data.dwcAttribute) { + record[data.dwcAttribute] = data.value + } } [record] diff --git a/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy b/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy index 68bc3277e..ff07b372d 100644 --- a/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy +++ b/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy @@ -335,7 +335,7 @@ class ParatooProtocolConfig { plotGeoJson } - static Map createConvexHullGeoJSON (List features, String name, String externalId = "", String notes = "") { + static Map createConvexHullGeoJSON (List features, String name, String externalId = "", String notes = "", String description = "") { features = features.findAll { it.geometry != null } List featureGeometries = features.collect { it.geometry } Geometry geometry = GeometryUtils.getFeatureCollectionConvexHull(featureGeometries) @@ -346,7 +346,7 @@ class ParatooProtocolConfig { name: name, externalId: externalId, notes: notes, - description: "${name} (convex hull of all features)", + description: "${description?:name}", ], features: features ] diff --git a/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy b/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy index 47aebe431..d794958cc 100644 --- a/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy +++ b/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy @@ -12,9 +12,6 @@ import org.codehaus.jackson.map.ObjectMapper import org.grails.web.converters.marshaller.json.CollectionMarshaller import org.grails.web.converters.marshaller.json.MapMarshaller -import java.time.format.DateTimeTextProvider -import java.time.temporal.TemporalField - import static grails.async.Promises.waitAll /** * Tests for the ParatooService. @@ -82,6 +79,7 @@ class ParatooServiceSpec extends MongoSpec implements ServiceUnitTest", + "version_core": "" + ] + ) + ParatooCollectionId paratooCollectionId = buildCollectionId("mintCollectionIdBasalAreaPayload","guid-3") + Map dataSet = [dataSetId:'d1', grantId:'g1', surveyId:paratooCollectionId.toMap()] + ParatooProject project = new ParatooProject(id: projectId, project: new Project(projectId: projectId, custom: [dataSets: [dataSet]])) + Map surveyData = readSurveyData('basalAreaDbhReverseLookup') + Map site + + when: + Map result = service.submitCollection(collection, project) + waitAll(result.promise) + println ("finished waiting") + + then: + 1 * webService.doPost(*_) >> [resp: surveyData] + 1 * tokenService.getAuthToken(true) >> Mock(AccessToken) + 2 * projectService.updateDataSet(projectId, _) >> [status: 'ok'] + 0 * siteService.create(_) + 1 * activityService.create({ + it.startDate == "2023-09-22T00:59:47Z" && it.endDate == "2023-09-23T00:59:47Z" && + it.plannedStartDate == "2023-09-22T00:59:47Z" && it.plannedEndDate == "2023-09-23T00:59:47Z" && + it.externalIds[0].externalId == "d1" && it.externalIds[0].idType == ExternalId.IdType.MONITOR_MINTED_COLLECTION_ID + }) >> [activityId: '123'] + 1 * recordService.getAllByActivity('123') >> [] + 1 * settingService.getSetting('paratoo.surveyData.mapping') >> { + (["guid-3": [ + "name" : "Basal Area - DBH", + "usesPlotLayout": true, + "tags" : ["survey"], + "apiEndpoint" : "basal-area-dbh-measure-surveys", + "overrides" : [ + "dataModel": null, + "viewModel": null + ] + ]] as JSON).toString() + } + 1 * userService.getCurrentUserDetails() >> [userId: userId] + 1 * userService.setCurrentUser(userId) + + when: + String date = DateUtil.format(new Date()) + date = date.replace("Z", ".999Z") + surveyData["collections"]["basal-area-dbh-measure-survey"]["plot_visit"]["plot_layout"]["updatedAt"] = [date] + result = service.submitCollection(collection, project) + + then: + 1 * webService.doPost(*_) >> [resp: surveyData] + 1 * tokenService.getAuthToken(true) >> Mock(AccessToken) + 2 * projectService.updateDataSet(projectId, _) >> [status: 'ok'] + 1 * siteService.create(_) >> { site = it[0]; [siteId: 's1'] } + 1 * activityService.create({ + it.startDate == "2023-09-22T00:59:47Z" && it.endDate == "2023-09-23T00:59:47Z" && + it.plannedStartDate == "2023-09-22T00:59:47Z" && it.plannedEndDate == "2023-09-23T00:59:47Z" && + it.externalIds[0].externalId == "d1" && it.externalIds[0].idType == ExternalId.IdType.MONITOR_MINTED_COLLECTION_ID + }) >> [activityId: '123'] + 1 * recordService.getAllByActivity('123') >> [] + 1 * settingService.getSetting('paratoo.surveyData.mapping') >> { + (["guid-3": [ + "name" : "Basal Area - DBH", + "usesPlotLayout": true, + "tags" : ["survey"], + "apiEndpoint" : "basal-area-dbh-measure-surveys", + "overrides" : [ + "dataModel": null, + "viewModel": null + ] + ]] as JSON).toString() + } + 1 * userService.getCurrentUserDetails() >> [userId: userId] + 1 * userService.setCurrentUser(userId) + result.updateResult == [status: 'ok'] + } + + void "isUpdatedPlotLayout should check plot layout has been updated after site has been updated" () { + given: + def date1 = DateUtil.parseWithMilliseconds("2023-09-22T00:59:47.111Z") + def date2 = DateUtil.parseWithMilliseconds("2023-09-23T00:59:47.111Z") + def date3 = DateUtil.parseWithMilliseconds("2023-09-24T00:59:47.111Z") + + when: + def result = service.isUpdatedPlotLayout(date1, date2) + + then: + result + + when: + result = service.isUpdatedPlotLayout(date3, date2) + + then: + !result + } + private Map getProject(){ [ projectId:"p1", diff --git a/src/test/groovy/au/org/ala/ecodata/SiteServiceSpec.groovy b/src/test/groovy/au/org/ala/ecodata/SiteServiceSpec.groovy index 1d73c1805..cc81daa90 100644 --- a/src/test/groovy/au/org/ala/ecodata/SiteServiceSpec.groovy +++ b/src/test/groovy/au/org/ala/ecodata/SiteServiceSpec.groovy @@ -4,13 +4,12 @@ import com.mongodb.BasicDBObject import grails.converters.JSON import grails.test.mongodb.MongoSpec import grails.testing.services.ServiceUnitTest +import org.grails.web.converters.marshaller.json.CollectionMarshaller /*import grails.test.mixin.TestMixin import grails.test.mixin.mongodb.MongoDbTestMixin*/ -import org.grails.web.converters.marshaller.json.CollectionMarshaller -import org.grails.web.converters.marshaller.json.MapMarshaller -import spock.lang.Specification +import org.grails.web.converters.marshaller.json.MapMarshaller /** * Specification / tests for the SiteService */ @@ -288,6 +287,24 @@ class SiteServiceSpec extends MongoSpec implements ServiceUnitTest } + def "Sites can be listed by externalId and sorted"() { + when: + def result + Site.withSession { session -> + result = service.create([name:'Site 1', siteId:"s1", externalIds:[new ExternalId(externalId:'e1', idType:ExternalId.IdType.MONITOR_PLOT_GUID)]]) + session.flush() + result = service.create([name:'Site 2', siteId:"s2", externalIds:[new ExternalId(externalId:'e1', idType:ExternalId.IdType.MONITOR_PLOT_GUID)]]) + session.flush() + } + then: + def sites = Site.findAllByExternalId(ExternalId.IdType.MONITOR_PLOT_GUID, 'e1', ['sort': "lastUpdated", 'order': "desc"]) + sites.size() == 2 + sites[0].name == 'Site 2' + sites[0].externalIds.size() == 1 + sites.externalIds.externalId == [['e1'], ['e1']] + sites.externalIds.idType == [[ExternalId.IdType.MONITOR_PLOT_GUID], [ExternalId.IdType.MONITOR_PLOT_GUID]] + } + private Map buildExtent(source, type, coordinates, pid = '') { return [source:source, geometry:[type:type, coordinates: coordinates, pid:pid]] diff --git a/src/test/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfigSpec.groovy b/src/test/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfigSpec.groovy index 651031e85..f42b26cf5 100644 --- a/src/test/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfigSpec.groovy +++ b/src/test/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfigSpec.groovy @@ -196,7 +196,7 @@ class ParatooProtocolConfigSpec extends Specification { coordinates:[[[138.6372, -34.9723], [138.6371, -34.9723], [138.6371, -34.9714], [138.6382, -34.9714], [138.6383, -34.9714], [138.6383, -34.9723], [138.6372, -34.9723]]], type:"Polygon" ], - properties:[name:"SATFLB0001 - Control (100 x 100)", externalId:"2", notes:"Core monitoring plot some comment", description:"SATFLB0001 - Control (100 x 100) (convex hull of all features)"], + properties:[name:"SATFLB0001 - Control (100 x 100)", externalId:"2", notes:"Core monitoring plot some comment", description:"SATFLB0001 - Control (100 x 100)"], features:[ [ type:"Feature", @@ -312,7 +312,7 @@ class ParatooProtocolConfigSpec extends Specification { features : [[type: "Feature", geometry: [type: "Point", coordinates: [138.63, -35.0005]], properties:[name:"Point aParatooForm 1-1", externalId:40, id:"aParatooForm 1-1"]]], properties: [ name : "aParatooForm 1 site - ${startDateInDefaultTimeZone}", - description: "aParatooForm 1 site - ${startDateInDefaultTimeZone} (convex hull of all features)", + description: "aParatooForm 1 site - ${startDateInDefaultTimeZone}", externalId: "", notes: "", ] @@ -348,6 +348,83 @@ class ParatooProtocolConfigSpec extends Specification { ] } + def "getPlotLayoutUpdatedAt should get plot layout's updatedAt property" () { + given: + def surveyData = readSurveyData('floristicsStandardReverseLookup') + Map floristicsSurveyConfig = [ + apiEndpoint:'floristics-veg-survey-lites', + usesPlotLayout:true + ] + ParatooProtocolConfig config = new ParatooProtocolConfig(floristicsSurveyConfig) + config.setSurveyId(ParatooCollectionId.fromMap([survey_metadata: surveyData.survey_metadata])) + + when: + def result = config.getPlotLayoutUpdatedAt(surveyData.collections) + + then: + result.equals(DateUtil.parseWithMilliseconds("2023-09-14T06:00:11.473Z")) + } + + def "createConvexHullGeoJSON should create convex hull based on valid geometries" () { + + given: + def features = [ + [ + type: "Feature", + geometry: [ + type: "Polygon", + coordinates: [[[1, 1], [1, 2], [2, 2], [2, 1], [1, 1]]] + ] + ], + [ + type: "Feature", + geometry: [ + type: "Polygon", + coordinates: [[[1, 1], [1, 2], [2, 2], [2, 1], [1, 1]]] + ] + ], + [ + type: "Feature", + geometry: null + ] + ] + + when: + def result = ParatooProtocolConfig.createConvexHullGeoJSON(features, "test name", "1", "test notes") + + then: + result == [ + type: "Feature", + geometry: [ + type: "Polygon", + coordinates: [[[1, 1], [1, 2], [2, 2], [2, 1], [1, 1]]] + ], + properties: [ + name: "test name", + externalId: "1", + description: "test name", + notes: "test notes" + ], + features:[ + [ + type: "Feature", + geometry: [ + type: "Polygon", + coordinates: [[[1, 1], [1, 2], [2, 2], [2, 1], [1, 1]]] + ] + ], + [ + type: "Feature", + geometry: [ + type: "Polygon", + coordinates: [[[1, 1], [1, 2], [2, 2], [2, 1], [1, 1]]] + ] + ] + ] + ] + + } + def transformData(Map surveyDataAndObservations, ActivityForm form, ParatooProtocolConfig config) { ParatooService.addPlotDataToObservations(surveyDataAndObservations, config) paratooService.rearrangeSurveyData(surveyDataAndObservations, surveyDataAndObservations, form.sections[0].template.relationships.ecodata, form.sections[0].template.relationships.apiOutput) From 55254b6f5e35bee2166fdb899301995f84bc32a3 Mon Sep 17 00:00:00 2001 From: temi Date: Thu, 16 May 2024 14:56:16 +1000 Subject: [PATCH 26/44] #941 - fixes broken test --- .../org/ala/ecodata/ParatooServiceSpec.groovy | 45 ++++++++++++++++--- 1 file changed, 40 insertions(+), 5 deletions(-) diff --git a/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy b/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy index d794958cc..2d6fd58c2 100644 --- a/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy +++ b/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy @@ -348,7 +348,7 @@ class ParatooServiceSpec extends MongoSpec implements ServiceUnitTest> [userId: userId] 1 * userService.setCurrentUser(userId) + } - when: + void "The service will create a new site if plot layout has been updated" () { + setup: + String projectId = 'p1' + String orgMintedId = 'd1' + Date afterSubmissionDate = DateUtil.parseWithMilliseconds("2023-09-15T06:00:11.996Z") + Date beforeSubmissionDate = DateUtil.parseWithMilliseconds("2023-09-14T06:00:11.996Z") + Site.withSession { session -> + new Site( + name: "SATFLB0001 - Control (100 x 100)", + siteId: "s0", + extent: [geometry: DUMMY_POLYGON], + description: "SATFLB0001 - Control (100 x 100)", + notes: "Core monitoring plot some comment", + type: "compound", + externalIds: [new ExternalId(externalId: "2", idType: ExternalId.IdType.MONITOR_PLOT_GUID)], + dateCreated: afterSubmissionDate, + lastUpdated: afterSubmissionDate + ).save(flush: true) + session.flush() + } + ParatooProtocolId protocol = new ParatooProtocolId(id: "1", version: 1) + ParatooCollection collection = new ParatooCollection( + orgMintedUUID:orgMintedId, + coreProvenance: [ + "system_core": "", + "version_core": "" + ] + ) + ParatooCollectionId paratooCollectionId = buildCollectionId("mintCollectionIdBasalAreaPayload","guid-3") + Map dataSet = [dataSetId:'d1', grantId:'g1', surveyId:paratooCollectionId.toMap()] + ParatooProject project = new ParatooProject(id: projectId, project: new Project(projectId: projectId, custom: [dataSets: [dataSet]])) + Map surveyData = readSurveyData('basalAreaDbhReverseLookup') String date = DateUtil.format(new Date()) date = date.replace("Z", ".999Z") - surveyData["collections"]["basal-area-dbh-measure-survey"]["plot_visit"]["plot_layout"]["updatedAt"] = [date] - result = service.submitCollection(collection, project) + surveyData.collections."basal-area-dbh-measure-survey"."plot_visit"."plot_layout"."updatedAt" = date + Map site + + when: + def result = service.submitCollection(collection, project) + waitAll(result.promise) then: 1 * webService.doPost(*_) >> [resp: surveyData] From 71866f464fe0a6cdf23fa4929691aebe97df76e7 Mon Sep 17 00:00:00 2001 From: chrisala Date: Fri, 17 May 2024 08:44:46 +1000 Subject: [PATCH 27/44] Don't include site name in data set name for non-plot protocols #942 --- .../services/au/org/ala/ecodata/ParatooService.groovy | 9 +++++---- .../groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy | 8 ++++---- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/grails-app/services/au/org/ala/ecodata/ParatooService.groovy b/grails-app/services/au/org/ala/ecodata/ParatooService.groovy index f0c5ced05..2a5be244b 100644 --- a/grails-app/services/au/org/ala/ecodata/ParatooService.groovy +++ b/grails-app/services/au/org/ala/ecodata/ParatooService.groovy @@ -299,8 +299,8 @@ class ParatooService { dataSet.format = DATASET_DATABASE_TABLE dataSet.sizeUnknown = true // Update the data set name as the information supplied during /mint-identifier isn't enough - // to ensure uniqueness - dataSet.name = buildUpdatedDataSetSummaryName(siteName, dataSet.startDate, dataSet.endDate, form.name, surveyId) + // to ensure uniqueness. + dataSet.name = buildUpdatedDataSetSummaryName(siteName, dataSet.startDate, dataSet.endDate, form.name, surveyId, config) // Delete previously created activity so that duplicate species records are not created. // Updating existing activity will also create duplicates since it relies on outputSpeciesId to determine @@ -319,9 +319,10 @@ class ParatooService { } } - protected static String buildUpdatedDataSetSummaryName(String siteName, String startDate, String endDate, String protocolName, ParatooCollectionId surveyId) { + protected static String buildUpdatedDataSetSummaryName(String siteName, String startDate, String endDate, String protocolName, ParatooCollectionId surveyId, ParatooProtocolConfig config) { String name = protocolName - if (siteName) { + //The site name for non-plot based data sets is not used as it contains the same data (protocol and time) as the name + if (siteName && config.usesPlotLayout) { name += " (" + siteName + ")" } if (startDate && endDate && startDate != endDate) { diff --git a/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy b/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy index 2d6fd58c2..18b2fb6a5 100644 --- a/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy +++ b/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy @@ -1494,10 +1494,10 @@ class ParatooServiceSpec extends MongoSpec implements ServiceUnitTest Date: Fri, 17 May 2024 13:45:06 +1000 Subject: [PATCH 28/44] #951 - checks species match using name matching server --- grails-app/conf/application.groovy | 1 + .../au/org/ala/ecodata/ParatooService.groovy | 33 +++++---- .../ala/ecodata/SpeciesReMatchService.groovy | 32 ++++++++ .../org/ala/ecodata/ParatooServiceSpec.groovy | 73 +++++++++++-------- 4 files changed, 97 insertions(+), 42 deletions(-) diff --git a/grails-app/conf/application.groovy b/grails-app/conf/application.groovy index 3f0209f77..d610a3105 100644 --- a/grails-app/conf/application.groovy +++ b/grails-app/conf/application.groovy @@ -503,6 +503,7 @@ if (!ala.baseURL) { } bie.ws.url = "https://bie-ws.ala.org.au/" bie.url = "https://bie.ala.org.au/" +namesmatching.url = "https://namematching-ws-test.ala.org.au/" if (!collectory.baseURL) { //collectory.baseURL = "https://collectory-dev.ala.org.au/" diff --git a/grails-app/services/au/org/ala/ecodata/ParatooService.groovy b/grails-app/services/au/org/ala/ecodata/ParatooService.groovy index f0c5ced05..d35a2a46e 100644 --- a/grails-app/services/au/org/ala/ecodata/ParatooService.groovy +++ b/grails-app/services/au/org/ala/ecodata/ParatooService.groovy @@ -77,6 +77,7 @@ class ParatooService { RecordService recordService MetadataService metadataService UserService userService + SpeciesReMatchService speciesReMatchService /** * The rules we use to find projects eligible for use by paratoo are: @@ -2053,7 +2054,7 @@ class ParatooService { /** * Transforms a species name to species object used by ecodata. * e.g. Acacia glauca [Species] (scientific: Acacia glauca Willd.) - * [name: "Acacia glauca Willd.", scientificName: "Acacia glauca Willd.", guid: "A_GUID"] + * [name: "Acacia glauca Willd.", scientificName: "Acacia glauca Willd.", commonName: "Acacia glauca", guid: "A_GUID"] * Guid is necessary to generate species occurrence record. Guid is found by searching the species name with BIE. If not found, then a default value is added. * @param name * @return @@ -2064,28 +2065,34 @@ class ParatooService { } String regex = "([^\\[\\(]*)(?:\\[(.*)\\])?\\s*(?:\\(scientific:\\s*(.*?)\\))?" + String commonName, scientificName = name Pattern pattern = Pattern.compile(regex) Matcher matcher = pattern.matcher(name) - Map result = [name: name, scientificName: name, commonName: name, outputSpeciesId: UUID.randomUUID().toString()] + Map result = [scientificName: name, commonName: name, outputSpeciesId: UUID.randomUUID().toString()] if (matcher.find()) { - String commonName = matcher.group(1)?.trim() - String scientificName = matcher.group(3)?.trim() - result.commonName = commonName ?: result.commonName + commonName = matcher.group(1)?.trim() + scientificName = matcher.group(3)?.trim() result.taxonRank = matcher.group(2)?.trim() - result.scientificName = scientificName ?: commonName ?: result.scientificName - result.name = scientificName ?: commonName ?: result.name + result.scientificName = scientificName + result.commonName = commonName } - metadataService.autoPopulateSpeciesData(result) + Map resp = speciesReMatchService.searchByName(scientificName) + if (resp) { + result.putAll(resp) + } // try again with common name - if ((result.guid == null) && result.commonName) { - def speciesObject = [scientificName: result.commonName] - metadataService.autoPopulateSpeciesData(speciesObject) - result.guid = speciesObject.guid - result.scientificName = result.scientificName ?: speciesObject.scientificName + if ((result.guid == null) && commonName) { + resp = speciesReMatchService.searchByName(commonName) + if (resp) { + result.putAll(resp) + result.commonName = commonName + } } + result.name = result.commonName ? result.scientificName ? "${result.scientificName} (${result.commonName})" : result.commonName : result.scientificName + // record is only created if guid is present result.guid = result.guid ?: Record.UNMATCHED_GUID result diff --git a/grails-app/services/au/org/ala/ecodata/SpeciesReMatchService.groovy b/grails-app/services/au/org/ala/ecodata/SpeciesReMatchService.groovy index 11214a11d..a88af4a85 100644 --- a/grails-app/services/au/org/ala/ecodata/SpeciesReMatchService.groovy +++ b/grails-app/services/au/org/ala/ecodata/SpeciesReMatchService.groovy @@ -90,4 +90,36 @@ class SpeciesReMatchService { webService.getJson(url) }) } + + Map searchByName (String name, boolean addDetails = false) { + Map result = searchNameMatchingServer(name) + if (result) { + Map resp = [ + scientificName: result.scientificName, + commonName: result.vernacularName, + guid: result.taxonConceptID, + taxonRank: result.rank + ] + + if(addDetails) { + resp.put('details', result) + } + + return resp + } + } + + Map searchNameMatchingServer(String name) { + name = name?.toLowerCase() ?: "" + cacheService.get('name-matching-server-' + name, { + def encodedQuery = URLEncoder.encode(name ?: '', "UTF-8") + def url = "${grailsApplication.config.getProperty('namesmatching.url')}api/search?q=${encodedQuery}" + def resp = webService.getJson(url) + if (!resp.success) { + return null + } + + resp + }) + } } diff --git a/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy b/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy index 2d6fd58c2..99c9dee9a 100644 --- a/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy +++ b/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy @@ -26,6 +26,7 @@ class ParatooServiceSpec extends MongoSpec implements ServiceUnitTest> null - - when: // no scientific name - result = service.transformSpeciesName("Frogs [Class] (scientific: )") - outputSpeciesId = result.remove("outputSpeciesId") - - then: - outputSpeciesId != null - result == [name: "Frogs", scientificName: "Frogs", guid: "A_GUID", commonName: "Frogs", taxonRank: "Class"] - 2 * metadataService.autoPopulateSpeciesData(_) >> null - } - void "buildRelationshipTree should build relationship tree correctly"() { given: def properties = [ @@ -723,6 +706,25 @@ class ParatooServiceSpec extends MongoSpec implements ServiceUnitTest> null + + when: // no scientific name + result = service.transformSpeciesName("Frogs [Class] (scientific: )") + outputSpeciesId = result.remove("outputSpeciesId") + + then: + outputSpeciesId != null + result == [name: "Frogs", scientificName: "", guid: "A_GUID", commonName: "Frogs", taxonRank: "Class"] + 2 * speciesReMatchService.searchByName(_) >> null + } + void "buildTreeFromParentChildRelationships should build tree correctly"() { given: def relationships = [ @@ -1399,13 +1401,19 @@ class ParatooServiceSpec extends MongoSpec implements ServiceUnitTest> [ + commonName: "Cat", + scientificName: "Felis catus", + guid: "TAXON_ID", + taxonRank: "species" + ] result == [ lut: [ commonName: "Cat", - name: "Cat", - taxonRank: null, - scientificName: "Cat", - guid: "A_GUID" + name: "Felis catus (Cat)", + taxonRank: "species", + scientificName: "Felis catus", + guid: "TAXON_ID" ] ] @@ -1417,18 +1425,25 @@ class ParatooServiceSpec extends MongoSpec implements ServiceUnitTest> null + 1 * speciesReMatchService.searchByName("Cats") >> [ + commonName: "Cat", + scientificName: "Felis catus", + guid: "TAXON_ID", + taxonRank: "species" + ] result == [ lut: [ - commonName: "Cat", - name: "Cat", - taxonRank: null, - scientificName: "Cat", - guid: "A_GUID" + commonName: "Cats", + name: "Felis catus (Cats)", + taxonRank: "species", + scientificName: "Felis catus", + guid: "TAXON_ID" ] ] } From 62ad6150c4e9cbc1fa831c74b1162463078bca98 Mon Sep 17 00:00:00 2001 From: temi Date: Mon, 20 May 2024 07:06:56 +1000 Subject: [PATCH 29/44] #951 - adds vernacular name search --- grails-app/conf/application.groovy | 1 + .../au/org/ala/ecodata/ParatooService.groovy | 2 +- .../ala/ecodata/SpeciesReMatchService.groovy | 25 ++++++- .../ecodata/SpeciesReMatchServiceSpec.groovy | 65 +++++++++++++++++++ 4 files changed, 89 insertions(+), 4 deletions(-) diff --git a/grails-app/conf/application.groovy b/grails-app/conf/application.groovy index d610a3105..da139f946 100644 --- a/grails-app/conf/application.groovy +++ b/grails-app/conf/application.groovy @@ -504,6 +504,7 @@ if (!ala.baseURL) { bie.ws.url = "https://bie-ws.ala.org.au/" bie.url = "https://bie.ala.org.au/" namesmatching.url = "https://namematching-ws-test.ala.org.au/" +namematching.strategy = ["exactMatch", "vernacularMatch"] if (!collectory.baseURL) { //collectory.baseURL = "https://collectory-dev.ala.org.au/" diff --git a/grails-app/services/au/org/ala/ecodata/ParatooService.groovy b/grails-app/services/au/org/ala/ecodata/ParatooService.groovy index d35a2a46e..577fb5df5 100644 --- a/grails-app/services/au/org/ala/ecodata/ParatooService.groovy +++ b/grails-app/services/au/org/ala/ecodata/ParatooService.groovy @@ -2084,7 +2084,7 @@ class ParatooService { } // try again with common name if ((result.guid == null) && commonName) { - resp = speciesReMatchService.searchByName(commonName) + resp = speciesReMatchService.searchByName(commonName, false, true) if (resp) { result.putAll(resp) result.commonName = commonName diff --git a/grails-app/services/au/org/ala/ecodata/SpeciesReMatchService.groovy b/grails-app/services/au/org/ala/ecodata/SpeciesReMatchService.groovy index a88af4a85..b7fb38aad 100644 --- a/grails-app/services/au/org/ala/ecodata/SpeciesReMatchService.groovy +++ b/grails-app/services/au/org/ala/ecodata/SpeciesReMatchService.groovy @@ -91,9 +91,14 @@ class SpeciesReMatchService { }) } - Map searchByName (String name, boolean addDetails = false) { - Map result = searchNameMatchingServer(name) - if (result) { + Map searchByName (String name, boolean addDetails = false, boolean useVernacularSearch = false ) { + Map result + if (useVernacularSearch) + result = searchNameMatchingServer(name) + else + result = searchByVernacularNameOnNameMatchingServer(name) + List strategy = grailsApplication.config.getProperty('namematching.strategy', List) + if (strategy.contains(result?.matchType)) { Map resp = [ scientificName: result.scientificName, commonName: result.vernacularName, @@ -122,4 +127,18 @@ class SpeciesReMatchService { resp }) } + + Map searchByVernacularNameOnNameMatchingServer (String name) { + name = name?.toLowerCase() ?: "" + cacheService.get('name-matching-server-vernacular-name' + name, { + def encodedQuery = URLEncoder.encode(name ?: '', "UTF-8") + def url = "${grailsApplication.config.getProperty('namesmatching.url')}api/searchByVernacularName?vernacularName=${encodedQuery}" + def resp = webService.getJson(url) + if (!resp.success) { + return null + } + + resp + }) + } } diff --git a/src/test/groovy/au/org/ala/ecodata/SpeciesReMatchServiceSpec.groovy b/src/test/groovy/au/org/ala/ecodata/SpeciesReMatchServiceSpec.groovy index c848689b5..09dbddcde 100644 --- a/src/test/groovy/au/org/ala/ecodata/SpeciesReMatchServiceSpec.groovy +++ b/src/test/groovy/au/org/ala/ecodata/SpeciesReMatchServiceSpec.groovy @@ -67,4 +67,69 @@ class SpeciesReMatchServiceSpec extends Specification implements ServiceUnitTest result2 == resp2 } + void "search name server by name" () { + setup: + grailsApplication.config.namesmatching.url = "http://localhost:8080/" + grailsApplication.config.namesmatching.strategy = ["exactMatch", "vernacularMatch"] + def resp = [ + "success": true, + "scientificName": "Red", + "taxonConceptID": "ALA_DR22913_1168_0", + "rank": "genus", + "rankID": 6000, + "lft": 24693, + "rgt": 24693, + "matchType": "higherMatch", + "nameType": "SCIENTIFIC", + "kingdom": "Bamfordvirae", + "kingdomID": "https://www.catalogueoflife.org/data/taxon/8TRHY", + "phylum": "Nucleocytoviricota", + "phylumID": "https://www.catalogueoflife.org/data/taxon/5G", + "classs": "Megaviricetes", + "classID": "https://www.catalogueoflife.org/data/taxon/6224M", + "order": "Pimascovirales", + "orderID": "https://www.catalogueoflife.org/data/taxon/623FC", + "family": "Iridoviridae", + "familyID": "https://www.catalogueoflife.org/data/taxon/BFM", + "genus": "Red", + "genusID": "ALA_DR22913_1168_0", + "issues": [ + "noIssue" + ] + ] + service.webService.getJson(_) >> resp + when: + def result = service.searchByName("name") + + then: + result == null + + when: + resp.matchType = "exactMatch" + def result2 = service.searchByName("name") + + then: + service.webService.getJson(_) >> resp + result2 == [ + scientificName: "Red", + commonName: null, + guid: "ALA_DR22913_1168_0", + taxonRank: "genus" + ] + + when: + resp.matchType = "vernacularMatch" + def result3 = service.searchByName("name", false, true) + + then: + service.webService.getJson(_) >> resp + result3 == [ + scientificName: "Red", + commonName: null, + guid: "ALA_DR22913_1168_0", + taxonRank: "genus" + ] + + } + } From 59fa0ba924009e266a6f4db1c5da409b153ef95e Mon Sep 17 00:00:00 2001 From: chrisala Date: Mon, 20 May 2024 07:55:46 +1000 Subject: [PATCH 30/44] Fixed unit tests #951 --- .../groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy b/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy index 99c9dee9a..fa735e97f 100644 --- a/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy +++ b/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy @@ -713,7 +713,8 @@ class ParatooServiceSpec extends MongoSpec implements ServiceUnitTest> null + 1 * speciesReMatchService.searchByName(_) >> null + 1 * speciesReMatchService.searchByName(_, false, true) >> null when: // no scientific name result = service.transformSpeciesName("Frogs [Class] (scientific: )") @@ -722,7 +723,8 @@ class ParatooServiceSpec extends MongoSpec implements ServiceUnitTest> null + 1 * speciesReMatchService.searchByName(_) >> null + 1 * speciesReMatchService.searchByName(_, false, true) >> null } void "buildTreeFromParentChildRelationships should build tree correctly"() { @@ -1431,7 +1433,7 @@ class ParatooServiceSpec extends MongoSpec implements ServiceUnitTest> null - 1 * speciesReMatchService.searchByName("Cats") >> [ + 1 * speciesReMatchService.searchByName("Cats", false, true) >> [ commonName: "Cat", scientificName: "Felis catus", guid: "TAXON_ID", From 2c6998835d41ab687081a24871f338be77bd1a39 Mon Sep 17 00:00:00 2001 From: temi Date: Mon, 20 May 2024 09:58:24 +1000 Subject: [PATCH 31/44] #951 - fixed a logic flaw --- .../au/org/ala/ecodata/SpeciesReMatchService.groovy | 2 +- .../au/org/ala/ecodata/SpeciesReMatchServiceSpec.groovy | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/grails-app/services/au/org/ala/ecodata/SpeciesReMatchService.groovy b/grails-app/services/au/org/ala/ecodata/SpeciesReMatchService.groovy index b7fb38aad..d02361bc5 100644 --- a/grails-app/services/au/org/ala/ecodata/SpeciesReMatchService.groovy +++ b/grails-app/services/au/org/ala/ecodata/SpeciesReMatchService.groovy @@ -93,7 +93,7 @@ class SpeciesReMatchService { Map searchByName (String name, boolean addDetails = false, boolean useVernacularSearch = false ) { Map result - if (useVernacularSearch) + if (!useVernacularSearch) result = searchNameMatchingServer(name) else result = searchByVernacularNameOnNameMatchingServer(name) diff --git a/src/test/groovy/au/org/ala/ecodata/SpeciesReMatchServiceSpec.groovy b/src/test/groovy/au/org/ala/ecodata/SpeciesReMatchServiceSpec.groovy index 09dbddcde..159c70f22 100644 --- a/src/test/groovy/au/org/ala/ecodata/SpeciesReMatchServiceSpec.groovy +++ b/src/test/groovy/au/org/ala/ecodata/SpeciesReMatchServiceSpec.groovy @@ -97,7 +97,7 @@ class SpeciesReMatchServiceSpec extends Specification implements ServiceUnitTest "noIssue" ] ] - service.webService.getJson(_) >> resp + service.webService.getJson({it.contains("search?q=")}) >> resp when: def result = service.searchByName("name") @@ -109,7 +109,7 @@ class SpeciesReMatchServiceSpec extends Specification implements ServiceUnitTest def result2 = service.searchByName("name") then: - service.webService.getJson(_) >> resp + service.webService.getJson({it.contains("search?q=")}) >> resp result2 == [ scientificName: "Red", commonName: null, @@ -122,7 +122,7 @@ class SpeciesReMatchServiceSpec extends Specification implements ServiceUnitTest def result3 = service.searchByName("name", false, true) then: - service.webService.getJson(_) >> resp + service.webService.getJson({it.contains("searchByVernacularName")}) >> resp result3 == [ scientificName: "Red", commonName: null, From 8ee8f6b8375ebb8243408a8caca3beb510960b3d Mon Sep 17 00:00:00 2001 From: chrisala Date: Tue, 21 May 2024 09:55:04 +1000 Subject: [PATCH 32/44] Fixed protocolCheck for projectParticipant role #956 --- .../au/org/ala/ecodata/ParatooService.groovy | 10 ++++-- .../org/ala/ecodata/ParatooServiceSpec.groovy | 32 ++++++++++++++++++- 2 files changed, 38 insertions(+), 4 deletions(-) diff --git a/grails-app/services/au/org/ala/ecodata/ParatooService.groovy b/grails-app/services/au/org/ala/ecodata/ParatooService.groovy index ce9246aef..c79b7179e 100644 --- a/grails-app/services/au/org/ala/ecodata/ParatooService.groovy +++ b/grails-app/services/au/org/ala/ecodata/ParatooService.groovy @@ -125,6 +125,7 @@ class ParatooService { private static List findProtocolsByCategories(List categories) { List forms = ActivityForm.findAllByCategoryInListAndExternalAndStatusNotEqual(categories, true, Status.DELETED) forms + forms } private List findUserProjects(String userId) { @@ -445,10 +446,13 @@ class ParatooService { private boolean protocolCheck(String userId, String projectId, String protocolId, boolean read) { List projects = userProjects(userId) + ParatooProject project = projects.find { it.id == projectId } - boolean protocol = project?.protocols?.find { it.externalIds.find { it.externalId == protocolId } } - int minimumAccess = read ? AccessLevel.projectParticipant.code : AccessLevel.editor.code - protocol && project.accessLevel.code >= minimumAccess + ActivityForm protocol = project?.protocols?.find { it.externalIds.find { it.externalId == protocolId } } + int minAccessLevel = AccessLevel.projectParticipant.code + // Note we don't need to include a check for ADMIN_ONLY_PROTOCOLS here as those protocol will have already be filtered + // out of the list of protocols attached to the project in findProjectProtocols if the user isn't an admin. + protocol && project.accessLevel.code >= minAccessLevel } Map findDataSet(String userId, String orgMintedUUID) { diff --git a/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy b/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy index c5116f3f4..1cfd60ac1 100644 --- a/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy +++ b/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy @@ -101,7 +101,7 @@ class ParatooServiceSpec extends MongoSpec implements ServiceUnitTest> DUMMY_POLYGON @@ -538,6 +538,10 @@ class ParatooServiceSpec extends MongoSpec implements ServiceUnitTest Date: Tue, 21 May 2024 10:52:05 +1000 Subject: [PATCH 33/44] Fixed typo #956 --- grails-app/services/au/org/ala/ecodata/ParatooService.groovy | 1 - 1 file changed, 1 deletion(-) diff --git a/grails-app/services/au/org/ala/ecodata/ParatooService.groovy b/grails-app/services/au/org/ala/ecodata/ParatooService.groovy index c79b7179e..b963f4d8f 100644 --- a/grails-app/services/au/org/ala/ecodata/ParatooService.groovy +++ b/grails-app/services/au/org/ala/ecodata/ParatooService.groovy @@ -125,7 +125,6 @@ class ParatooService { private static List findProtocolsByCategories(List categories) { List forms = ActivityForm.findAllByCategoryInListAndExternalAndStatusNotEqual(categories, true, Status.DELETED) forms - forms } private List findUserProjects(String userId) { From 81a2cb789162b473668f10258b4a44cdf1faba30 Mon Sep 17 00:00:00 2001 From: temi Date: Wed, 22 May 2024 15:35:57 +1000 Subject: [PATCH 34/44] #958 - added a flag to switch off record creation for certain protocols --- .../au/org/ala/ecodata/ParatooService.groovy | 3 +- .../paratoo/ParatooProtocolConfig.groovy | 1 + .../org/ala/ecodata/ParatooServiceSpec.groovy | 88 +++++++++++++++++++ 3 files changed, 91 insertions(+), 1 deletion(-) diff --git a/grails-app/services/au/org/ala/ecodata/ParatooService.groovy b/grails-app/services/au/org/ala/ecodata/ParatooService.groovy index b963f4d8f..d8b3077f2 100644 --- a/grails-app/services/au/org/ala/ecodata/ParatooService.groovy +++ b/grails-app/services/au/org/ala/ecodata/ParatooService.groovy @@ -1233,8 +1233,9 @@ class ParatooService { ArrayDeque modelVisitStack = new ArrayDeque<>() documentation = deepCopy(documentation) Map components = deepCopy(getComponents(documentation)) + boolean record = config.createSpeciesRecord - Map template = [dataModel: [], viewModel: [], modelName: capitalizeModelName(protocol.attributes.name), record: true, relationships: [ecodata: [:], apiOutput: [:]]] + Map template = [dataModel: [], viewModel: [], modelName: capitalizeModelName(protocol.attributes.name), record: record, relationships: [ecodata: [:], apiOutput: [:]]] Map properties = deepCopy(findProtocolEndpointDefinition(protocol, documentation)) if (properties == null) { throw new NotFoundException("No protocol endpoint found for ${protocol.attributes.endpointPrefix}/bulk") diff --git a/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy b/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy index ff07b372d..98a02dba7 100644 --- a/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy +++ b/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy @@ -17,6 +17,7 @@ class ParatooProtocolConfig { String name String apiEndpoint boolean usesPlotLayout = true + boolean createSpeciesRecord = true List tags String geometryType = 'Polygon' diff --git a/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy b/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy index 1cfd60ac1..83f2c4f5c 100644 --- a/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy +++ b/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy @@ -1549,6 +1549,94 @@ class ParatooServiceSpec extends MongoSpec implements ServiceUnitTest Date: Fri, 24 May 2024 16:35:01 +1000 Subject: [PATCH 35/44] Record Monitor activities as finished #961 --- grails-app/services/au/org/ala/ecodata/ParatooService.groovy | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/grails-app/services/au/org/ala/ecodata/ParatooService.groovy b/grails-app/services/au/org/ala/ecodata/ParatooService.groovy index d8b3077f2..158923b8e 100644 --- a/grails-app/services/au/org/ala/ecodata/ParatooService.groovy +++ b/grails-app/services/au/org/ala/ecodata/ParatooService.groovy @@ -479,12 +479,13 @@ class ParatooService { formVersion : activityForm.formVersion, description : "Activity submitted by monitor", projectId : collection.projectId, - publicationStatus: "published", + publicationStatus: PublicationStatus.PUBLISHED, siteId : dataSet.siteId, startDate : dataSet.startDate, endDate : dataSet.endDate, plannedStartDate : dataSet.startDate, plannedEndDate : dataSet.endDate, + progress : Activity.FINISHED, externalIds : [new ExternalId(idType: ExternalId.IdType.MONITOR_MINTED_COLLECTION_ID, externalId: dataSet.dataSetId)], userId : userId, outputs : [[ From 2ac26812b2880375bd4291b7b21b35683e581721 Mon Sep 17 00:00:00 2001 From: chrisala Date: Wed, 29 May 2024 12:33:11 +1000 Subject: [PATCH 36/44] Calculate area of compound site using features #962 --- .../au/org/ala/ecodata/SiteService.groovy | 14 ++++++- .../au/org/ala/ecodata/SiteServiceSpec.groovy | 37 +++++++++++++++++++ 2 files changed, 50 insertions(+), 1 deletion(-) diff --git a/grails-app/services/au/org/ala/ecodata/SiteService.groovy b/grails-app/services/au/org/ala/ecodata/SiteService.groovy index 4e5c746bc..37db8dbe7 100644 --- a/grails-app/services/au/org/ala/ecodata/SiteService.groovy +++ b/grails-app/services/au/org/ala/ecodata/SiteService.groovy @@ -623,7 +623,19 @@ class SiteService { def populateLocationMetadataForSite(Map site) { - def siteGeom = geometryAsGeoJson(site) + Map siteGeom + if (site.type == Site.TYPE_COMPOUND) { + siteGeom = [ + type:'GeometryCollection', + geometries: [ + site.features.collect{it.geometry} + ] + ] + } + else { + siteGeom = geometryAsGeoJson(site) + } + if (siteGeom) { GeometryJSON gjson = new GeometryJSON() Geometry geom = gjson.read((siteGeom as JSON).toString()) diff --git a/src/test/groovy/au/org/ala/ecodata/SiteServiceSpec.groovy b/src/test/groovy/au/org/ala/ecodata/SiteServiceSpec.groovy index cc81daa90..c3fe1bc5a 100644 --- a/src/test/groovy/au/org/ala/ecodata/SiteServiceSpec.groovy +++ b/src/test/groovy/au/org/ala/ecodata/SiteServiceSpec.groovy @@ -305,6 +305,43 @@ class SiteServiceSpec extends MongoSpec implements ServiceUnitTest sites.externalIds.idType == [[ExternalId.IdType.MONITOR_PLOT_GUID], [ExternalId.IdType.MONITOR_PLOT_GUID]] } + def "The site area is calculated from the FeatureCollection for a compound site"() { + setup: + def coordinates = [[148.260498046875, -37.26530995561874], [148.260498046875, -37.26531995561874], [148.310693359375, -37.26531995561874], [148.310693359375, -37.26531995561874], [148.260498046875, -37.26530995561874]] + def extent = buildExtent('drawn', 'Polygon', coordinates) + Map site = [type: Site.TYPE_COMPOUND, extent: extent, features: [ + [ + type : "Feature", + geometry: [ + type : "Polygon", + coordinates: coordinates + ] + ], + [ + type : "Feature", + geometry: [ + type : "Polygon", + coordinates: coordinates + ] + ] + ]] + + when: + service.populateLocationMetadataForSite(site) + + then: + 1 * spatialServiceMock.intersectGeometry(_, _) >> [:] + site.extent.geometry.aream2 == 4938.9846950349165d + + when: + site.type = Site.TYPE_WORKS_AREA + service.populateLocationMetadataForSite(site) + + then: + 1 * spatialServiceMock.intersectGeometry(_, _) >> [:] + site.extent.geometry.aream2 == 2469.492347517461 + + } private Map buildExtent(source, type, coordinates, pid = '') { return [source:source, geometry:[type:type, coordinates: coordinates, pid:pid]] From 90c92a54145c9d70d7eb79fe360b638eeeef12c6 Mon Sep 17 00:00:00 2001 From: chrisala Date: Wed, 29 May 2024 12:58:51 +1000 Subject: [PATCH 37/44] Change site name based on core/fauna #942 --- .../ecodata/paratoo/ParatooProtocolConfig.groovy | 10 +++++++--- .../paratoo/ParatooProtocolConfigSpec.groovy | 16 ++++++++-------- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy b/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy index 98a02dba7..09f805666 100644 --- a/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy +++ b/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy @@ -14,6 +14,8 @@ import org.locationtech.jts.geom.Geometry class ParatooProtocolConfig { static final String FAUNA_PLOT = 'Fauna plot' static final String CORE_PLOT = 'Core monitoring plot' + static final String FAUNA_PLOT_SHORT = 'Fauna' + static final String CORE_PLOT_SHORT = 'Core' String name String apiEndpoint boolean usesPlotLayout = true @@ -319,17 +321,19 @@ class ParatooProtocolConfig { Map plotSelection = getProperty(surveyData, plotSelectionPath) Map plotSelectionGeoJson = plotSelectionToGeoJson(plotSelection) - String plotLayoutDimensionLabel = getProperty(surveyData, plotLayoutDimensionLabelPath) String plotLayoutTypeLabel = getProperty(surveyData, plotLayoutTypeLabelPath) + List faunaPlotPoints = getProperty(surveyData, faunaPlotPointPath) - String name = plotSelectionGeoJson.properties.name + ' - ' + plotLayoutTypeLabel + ' (' + plotLayoutDimensionLabel + ')' + String name = plotSelectionGeoJson.properties.name + ' - ' + plotLayoutTypeLabel + ' ('+CORE_PLOT_SHORT+')' Map plotGeoJson = createFeatureFromGeoJSON(plotLayoutPoints, name, plotLayoutId, "${CORE_PLOT} ${plotSelectionGeoJson?.properties?.notes?:""}") - List faunaPlotPoints = getProperty(surveyData, faunaPlotPointPath) if (faunaPlotPoints) { + name = plotSelectionGeoJson.properties.name + ' - ' + plotLayoutTypeLabel + ' (' + FAUNA_PLOT_SHORT + ')' Map faunaPlotGeoJson = createFeatureFromGeoJSON(faunaPlotPoints, name, plotLayoutId, "${FAUNA_PLOT} ${plotSelectionGeoJson?.properties?.notes?:""}") List features = [plotGeoJson, faunaPlotGeoJson] + + name = plotSelectionGeoJson.properties.name + ' - ' + plotLayoutTypeLabel + ' (' + CORE_PLOT_SHORT + ' + ' + FAUNA_PLOT_SHORT + ')' plotGeoJson = createConvexHullGeoJSON(features, name, plotLayoutId, plotGeoJson.properties.notes) } diff --git a/src/test/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfigSpec.groovy b/src/test/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfigSpec.groovy index f42b26cf5..4a01b44e3 100644 --- a/src/test/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfigSpec.groovy +++ b/src/test/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfigSpec.groovy @@ -140,7 +140,7 @@ class ParatooProtocolConfigSpec extends Specification { type: "Polygon", coordinates: [[[152.880694, -27.388252], [152.880651, -27.388336], [152.880518, -27.388483], [152.880389, -27.388611], [152.88028, -27.388749], [152.880154, -27.388903], [152.880835, -27.389463], [152.880644, -27.389366], [152.880525, -27.389248], [152.88035, -27.389158], [152.880195, -27.389021], [152.880195, -27.389373], [152.880797, -27.388316], [152.881448, -27.388909], [152.881503, -27.388821], [152.881422, -27.388766], [152.881263, -27.388644], [152.881107, -27.388549], [152.880939, -27.388445], [152.881314, -27.389035], [152.88122, -27.389208], [152.881089, -27.389343], [152.880973, -27.389472], [152.880916, -27.389553], [152.880694, -27.388252]]] ], - properties: [name: "QDASEQ0001 - Control (100 x 100)", externalId: "1", description: "QDASEQ0001 - Control (100 x 100)", notes: "Core monitoring plot some comment"] + properties: [name: "QDASEQ0001 - Control (Core)", externalId: "1", description: "QDASEQ0001 - Control (Core)", notes: "Core monitoring plot some comment"] ] } @@ -196,7 +196,7 @@ class ParatooProtocolConfigSpec extends Specification { coordinates:[[[138.6372, -34.9723], [138.6371, -34.9723], [138.6371, -34.9714], [138.6382, -34.9714], [138.6383, -34.9714], [138.6383, -34.9723], [138.6372, -34.9723]]], type:"Polygon" ], - properties:[name:"SATFLB0001 - Control (100 x 100)", externalId:"2", notes:"Core monitoring plot some comment", description:"SATFLB0001 - Control (100 x 100)"], + properties:[name:"SATFLB0001 - Control (Core + Fauna)", externalId:"2", notes:"Core monitoring plot some comment", description:"SATFLB0001 - Control (Core + Fauna)"], features:[ [ type:"Feature", @@ -204,7 +204,7 @@ class ParatooProtocolConfigSpec extends Specification { type:"Polygon", coordinates:[[[138.63720760798054, -34.97222197296049], [138.63720760798054, -34.97204230990367], [138.63720760798054, -34.971862646846844], [138.63720760798054, -34.97168298379002], [138.63720760798054, -34.9715033207332], [138.63720760798054, -34.971413489204785], [138.63731723494544, -34.971413489204785], [138.6375364888752, -34.971413489204785], [138.63775574280498, -34.971413489204785], [138.63797499673475, -34.971413489204785], [138.63819425066453, -34.971413489204785], [138.63830387762943, -34.971413489204785], [138.63830387762943, -34.9715033207332], [138.63830387762943, -34.97168298379002], [138.63830387762943, -34.971862646846844], [138.63830387762943, -34.97204230990367], [138.63830387762943, -34.97222197296049], [138.63830387762943, -34.9723118044889], [138.63819425066453, -34.9723118044889], [138.63797499673475, -34.9723118044889], [138.63775574280498, -34.9723118044889], [138.6375364888752, -34.9723118044889], [138.63731723494544, -34.9723118044889], [138.63720760798054, -34.9723118044889], [138.63720760798054, -34.97222197296049]]] ], - properties:[name:"SATFLB0001 - Control (100 x 100)", externalId:"2", description:"SATFLB0001 - Control (100 x 100)", notes:"Core monitoring plot some comment"]], + properties:[name:"SATFLB0001 - Control (Core)", externalId:"2", description:"SATFLB0001 - Control (Core)", notes:"Core monitoring plot some comment"]], [ type:"Feature", geometry:[ @@ -212,7 +212,7 @@ class ParatooProtocolConfigSpec extends Specification { coordinates:[[[138.6371026907952, -34.971403261821905], [138.63709732396242, -34.972304399720215], [138.6381916652405, -34.972304399720215], [138.63819166764344, -34.9714076576406], [138.6371026907952, -34.971403261821905]]] ], properties:[ - name:"SATFLB0001 - Control (100 x 100)", externalId:"2", description:"SATFLB0001 - Control (100 x 100)", notes:"Fauna plot some comment" + name:"SATFLB0001 - Control (Fauna)", externalId:"2", description:"SATFLB0001 - Control (Fauna)", notes:"Fauna plot some comment" ] ] ] @@ -225,9 +225,9 @@ class ParatooProtocolConfigSpec extends Specification { coordinates:[[[138.63720760798054, -34.97222197296049], [138.63720760798054, -34.97204230990367], [138.63720760798054, -34.971862646846844], [138.63720760798054, -34.97168298379002], [138.63720760798054, -34.9715033207332], [138.63720760798054, -34.971413489204785], [138.63731723494544, -34.971413489204785], [138.6375364888752, -34.971413489204785], [138.63775574280498, -34.971413489204785], [138.63797499673475, -34.971413489204785], [138.63819425066453, -34.971413489204785], [138.63830387762943, -34.971413489204785], [138.63830387762943, -34.9715033207332], [138.63830387762943, -34.97168298379002], [138.63830387762943, -34.971862646846844], [138.63830387762943, -34.97204230990367], [138.63830387762943, -34.97222197296049], [138.63830387762943, -34.9723118044889], [138.63819425066453, -34.9723118044889], [138.63797499673475, -34.9723118044889], [138.63775574280498, -34.9723118044889], [138.6375364888752, -34.9723118044889], [138.63731723494544, -34.9723118044889], [138.63720760798054, -34.9723118044889], [138.63720760798054, -34.97222197296049]]] ], properties:[ - name:"SATFLB0001 - Control (100 x 100)", + name:"SATFLB0001 - Control (Core)", externalId:"2", - description:"SATFLB0001 - Control (100 x 100)", + description:"SATFLB0001 - Control (Core)", notes:"Core monitoring plot some comment" ] ], @@ -238,9 +238,9 @@ class ParatooProtocolConfigSpec extends Specification { coordinates:[[[138.6371026907952, -34.971403261821905], [138.63709732396242, -34.972304399720215], [138.6381916652405, -34.972304399720215], [138.63819166764344, -34.9714076576406], [138.6371026907952, -34.971403261821905]]] ], properties:[ - name:"SATFLB0001 - Control (100 x 100)", + name:"SATFLB0001 - Control (Fauna)", externalId:"2", - description:"SATFLB0001 - Control (100 x 100)", + description:"SATFLB0001 - Control (Fauna)", notes:"Fauna plot some comment" ] ] From 74af996c0710f494d1e9950fc7394dce71e95f20 Mon Sep 17 00:00:00 2001 From: chrisala Date: Wed, 29 May 2024 13:27:02 +1000 Subject: [PATCH 38/44] Fixed test #942 --- .../au/org/ala/ecodata/ParatooServiceSpec.groovy | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy b/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy index 83f2c4f5c..23501b396 100644 --- a/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy +++ b/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy @@ -338,8 +338,8 @@ class ParatooServiceSpec extends MongoSpec implements ServiceUnitTest new Site( - name: "SATFLB0001 - Control (100 x 100)", + name: "SATFLB0001 - Control (Core)", siteId: "s0", extent: [geometry: DUMMY_POLYGON], - description: "SATFLB0001 - Control (100 x 100)", + description: "SATFLB0001 - Control (Core)", notes: "Core monitoring plot some comment", type: "compound", externalIds: [new ExternalId(externalId: "2", idType: ExternalId.IdType.MONITOR_PLOT_GUID)], From b3dfa8cd81bad439418f97b8088cddcbba941016 Mon Sep 17 00:00:00 2001 From: temi Date: Wed, 29 May 2024 16:05:25 +1000 Subject: [PATCH 39/44] #965 - added dataSetId as external id for non-plot sites --- .../au/org/ala/ecodata/ParatooService.groovy | 13 +++++++++++-- .../org/ala/ecodata/ParatooServiceSpec.groovy | 17 ++++++++++++++--- 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/grails-app/services/au/org/ala/ecodata/ParatooService.groovy b/grails-app/services/au/org/ala/ecodata/ParatooService.groovy index d8b3077f2..c21beacd9 100644 --- a/grails-app/services/au/org/ala/ecodata/ParatooService.groovy +++ b/grails-app/services/au/org/ala/ecodata/ParatooService.groovy @@ -610,8 +610,17 @@ class ParatooService { siteProps.publicationStatus = PublicationStatus.PUBLISHED siteProps.projects = [project.projectId] String externalId = geoJson.properties?.externalId - if (externalId) { - siteProps.externalIds = [new ExternalId(idType: ExternalId.IdType.MONITOR_PLOT_GUID, externalId: externalId)] + if (config.usesPlotLayout) { + if (externalId) { + siteProps.externalIds = [new ExternalId(idType: ExternalId.IdType.MONITOR_PLOT_GUID, externalId: externalId)] + } + else { + log.error("No externalId found for plot layout for survey ${collection.orgMintedUUID}, project ${project.projectId}") + } + } + else { + // non-plot based data sets will have the dataSetId/orgMintedUUID as the external id + siteProps.externalIds = [new ExternalId(idType: ExternalId.IdType.MONITOR_PLOT_GUID, externalId: collection.orgMintedUUID)] } Site site // create new site for every non-plot submission diff --git a/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy b/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy index 83f2c4f5c..3272f699d 100644 --- a/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy +++ b/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy @@ -189,15 +189,16 @@ class ParatooServiceSpec extends MongoSpec implements ServiceUnitTest> [resp: [collections: ["coarse-woody-debris-survey": [uuid: "1", createdAt: "2023-09-01T00:00:00.123Z", start_date_time: "2023-09-01T00:00:00.123Z", end_date_time: "2023-09-01T00:00:00.123Z"]]]] + 1 * webService.doPost(*_) >> [resp: [collections: ["coarse-woody-debris-survey": [uuid: "1", createdAt: "2023-09-01T00:00:00.123Z", start_date_time: "2023-09-01T00:00:00.123Z", end_date_time: "2023-09-01T00:00:00.123Z"], "coarse-woody-debris-survey-observation": [[point: [lat: 1, lng: 2, name: [data: [attributes: [symbol: "ab"]]]]]]]]] 1 * tokenService.getAuthToken(true) >> Mock(AccessToken) 1 * projectService.updateDataSet(projectId, expectedDataSetAsync) >> [status: 'ok'] 1 * projectService.updateDataSet(projectId, expectedDataSetSync) >> [status: 'ok'] @@ -206,6 +207,7 @@ class ParatooServiceSpec extends MongoSpec implements ServiceUnitTest> [activityId: '123'] + 1 * siteService.create(_) >> { site = it[0]; [siteId: 's1'] } 1 * activityService.delete("123", true) >> [status: 'ok'] 1 * recordService.getAllByActivity('123') >> [] 1 * settingService.getSetting('paratoo.surveyData.mapping') >> { @@ -214,6 +216,7 @@ class ParatooServiceSpec extends MongoSpec implements ServiceUnitTest Date: Thu, 30 May 2024 11:10:41 +1000 Subject: [PATCH 40/44] Use different id type for plot selections #942 --- grails-app/domain/au/org/ala/ecodata/ExternalId.groovy | 2 +- .../services/au/org/ala/ecodata/ParatooService.groovy | 8 ++++---- .../groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/grails-app/domain/au/org/ala/ecodata/ExternalId.groovy b/grails-app/domain/au/org/ala/ecodata/ExternalId.groovy index 3257e5320..8ab58aed6 100644 --- a/grails-app/domain/au/org/ala/ecodata/ExternalId.groovy +++ b/grails-app/domain/au/org/ala/ecodata/ExternalId.groovy @@ -13,7 +13,7 @@ class ExternalId implements Comparable { enum IdType { INTERNAL_ORDER_NUMBER, TECH_ONE_CODE, WORK_ORDER, GRANT_AWARD, GRANT_OPPORTUNITY, RELATED_PROJECT, MONITOR_PROTOCOL_INTERNAL_ID, MONITOR_PROTOCOL_GUID, TECH_ONE_CONTRACT_NUMBER, MONITOR_PLOT_GUID, - MONITOR_MINTED_COLLECTION_ID, UNSPECIFIED } + MONITOR_PLOT_SELECTION_GUID, MONITOR_MINTED_COLLECTION_ID, UNSPECIFIED } static constraints = { } diff --git a/grails-app/services/au/org/ala/ecodata/ParatooService.groovy b/grails-app/services/au/org/ala/ecodata/ParatooService.groovy index c21beacd9..48d14c548 100644 --- a/grails-app/services/au/org/ala/ecodata/ParatooService.groovy +++ b/grails-app/services/au/org/ala/ecodata/ParatooService.groovy @@ -814,7 +814,7 @@ class ParatooService { // Monitor has users selecting a point as an approximate survey location then // laying out the plot using GPS when at the site. We only want to return the approximate planning // sites from this call - List plotSelections = sites.findAll{it.type == Site.TYPE_SURVEY_AREA && it.extent?.geometry?.type == 'Point'} + List plotSelections = sites.findAll{it.externalIds?.find{externalId -> externalId.idType == ExternalId.IdType.MONITOR_PLOT_SELECTION_GUID}} Map attributes = [ id:project.projectId, @@ -875,7 +875,7 @@ class ParatooService { // The project/s for the site will be specified by a subsequent call to /projects siteData.projects = [] - Site site = Site.findByExternalId(ExternalId.IdType.MONITOR_PLOT_GUID, siteData.externalId) + Site site = Site.findByExternalId(ExternalId.IdType.MONITOR_PLOT_SELECTION_GUID, siteData.externalId) Map result if (site) { result = siteService.update(siteData, site.siteId) @@ -892,7 +892,7 @@ class ParatooService { site.projects = [] // get all projects for the user I suppose - not sure why this isn't in the payload as it's in the UI... site.type = Site.TYPE_SURVEY_AREA - site.externalIds = [new ExternalId(idType: ExternalId.IdType.MONITOR_PLOT_GUID, externalId: geoJson.properties.externalId)] + site.externalIds = [new ExternalId(idType: ExternalId.IdType.MONITOR_PLOT_SELECTION_GUID, externalId: geoJson.properties.externalId)] site.publicationStatus = PublicationStatus.PUBLISHED // Mark the plot as read only as it is managed by the Monitor app @@ -919,7 +919,7 @@ class ParatooService { siteExternalIds.each { String siteExternalId -> - Site site = Site.findByExternalId(ExternalId.IdType.MONITOR_PLOT_GUID, siteExternalId) + Site site = Site.findByExternalId(ExternalId.IdType.MONITOR_PLOT_SELECTION_GUID, siteExternalId) if (site) { site.projects = site.projects ?: [] if (!site.projects.contains(project.id)) { diff --git a/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy b/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy index 3272f699d..9200d6428 100644 --- a/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy +++ b/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy @@ -239,7 +239,7 @@ class ParatooServiceSpec extends MongoSpec implements ServiceUnitTest> [projectArea, plot] From eb0bf90f0405603734881e2b2e2b908df08d875b Mon Sep 17 00:00:00 2001 From: chrisala Date: Thu, 30 May 2024 15:51:27 +1000 Subject: [PATCH 41/44] Added a tag to fauna/core plots AtlasOfLivingAustralia/fieldcapture#3204 --- .../ecodata/paratoo/ParatooProtocolConfig.groovy | 16 ++++++++++------ .../paratoo/ParatooProtocolConfigSpec.groovy | 12 +++++++----- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy b/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy index 09f805666..b8428020c 100644 --- a/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy +++ b/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy @@ -326,11 +326,11 @@ class ParatooProtocolConfig { String name = plotSelectionGeoJson.properties.name + ' - ' + plotLayoutTypeLabel + ' ('+CORE_PLOT_SHORT+')' - Map plotGeoJson = createFeatureFromGeoJSON(plotLayoutPoints, name, plotLayoutId, "${CORE_PLOT} ${plotSelectionGeoJson?.properties?.notes?:""}") + Map plotGeoJson = createFeatureFromGeoJSON(plotLayoutPoints, name, plotLayoutId, "${CORE_PLOT} ${plotSelectionGeoJson?.properties?.notes?:""}", CORE_PLOT_SHORT) if (faunaPlotPoints) { name = plotSelectionGeoJson.properties.name + ' - ' + plotLayoutTypeLabel + ' (' + FAUNA_PLOT_SHORT + ')' - Map faunaPlotGeoJson = createFeatureFromGeoJSON(faunaPlotPoints, name, plotLayoutId, "${FAUNA_PLOT} ${plotSelectionGeoJson?.properties?.notes?:""}") + Map faunaPlotGeoJson = createFeatureFromGeoJSON(faunaPlotPoints, name, plotLayoutId, "${FAUNA_PLOT} ${plotSelectionGeoJson?.properties?.notes?:""}", FAUNA_PLOT_SHORT) List features = [plotGeoJson, faunaPlotGeoJson] name = plotSelectionGeoJson.properties.name + ' - ' + plotLayoutTypeLabel + ' (' + CORE_PLOT_SHORT + ' + ' + FAUNA_PLOT_SHORT + ')' @@ -357,13 +357,13 @@ class ParatooProtocolConfig { ] } - static Map createFeatureFromGeoJSON(List plotLayoutPoints, String name, String plotLayoutId, String notes = "") { + static Map createFeatureFromGeoJSON(List plotLayoutPoints, String name, String plotLayoutId, String notes = "", String activityType = null) { Map plotGeometry = toGeometry(plotLayoutPoints) - createFeatureObject(plotGeometry, name, plotLayoutId, notes) + createFeatureObject(plotGeometry, name, plotLayoutId, notes, activityType) } - static Map createFeatureObject(Map plotGeometry, String name, String plotLayoutId, String notes = "") { - [ + static Map createFeatureObject(Map plotGeometry, String name, String plotLayoutId, String notes = "", String activityType = null) { + Map featureObject = [ type : 'Feature', geometry : plotGeometry, properties: [ @@ -373,6 +373,10 @@ class ParatooProtocolConfig { notes : notes ] ] + if (activityType) { + featureObject.properties.activityType = activityType + } + featureObject } static Map toGeometry(List points) { diff --git a/src/test/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfigSpec.groovy b/src/test/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfigSpec.groovy index 4a01b44e3..62d2b2efb 100644 --- a/src/test/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfigSpec.groovy +++ b/src/test/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfigSpec.groovy @@ -140,7 +140,7 @@ class ParatooProtocolConfigSpec extends Specification { type: "Polygon", coordinates: [[[152.880694, -27.388252], [152.880651, -27.388336], [152.880518, -27.388483], [152.880389, -27.388611], [152.88028, -27.388749], [152.880154, -27.388903], [152.880835, -27.389463], [152.880644, -27.389366], [152.880525, -27.389248], [152.88035, -27.389158], [152.880195, -27.389021], [152.880195, -27.389373], [152.880797, -27.388316], [152.881448, -27.388909], [152.881503, -27.388821], [152.881422, -27.388766], [152.881263, -27.388644], [152.881107, -27.388549], [152.880939, -27.388445], [152.881314, -27.389035], [152.88122, -27.389208], [152.881089, -27.389343], [152.880973, -27.389472], [152.880916, -27.389553], [152.880694, -27.388252]]] ], - properties: [name: "QDASEQ0001 - Control (Core)", externalId: "1", description: "QDASEQ0001 - Control (Core)", notes: "Core monitoring plot some comment"] + properties: [name: "QDASEQ0001 - Control (Core)", externalId: "1", description: "QDASEQ0001 - Control (Core)", notes: "Core monitoring plot some comment", activityType:'Core'] ] } @@ -204,7 +204,7 @@ class ParatooProtocolConfigSpec extends Specification { type:"Polygon", coordinates:[[[138.63720760798054, -34.97222197296049], [138.63720760798054, -34.97204230990367], [138.63720760798054, -34.971862646846844], [138.63720760798054, -34.97168298379002], [138.63720760798054, -34.9715033207332], [138.63720760798054, -34.971413489204785], [138.63731723494544, -34.971413489204785], [138.6375364888752, -34.971413489204785], [138.63775574280498, -34.971413489204785], [138.63797499673475, -34.971413489204785], [138.63819425066453, -34.971413489204785], [138.63830387762943, -34.971413489204785], [138.63830387762943, -34.9715033207332], [138.63830387762943, -34.97168298379002], [138.63830387762943, -34.971862646846844], [138.63830387762943, -34.97204230990367], [138.63830387762943, -34.97222197296049], [138.63830387762943, -34.9723118044889], [138.63819425066453, -34.9723118044889], [138.63797499673475, -34.9723118044889], [138.63775574280498, -34.9723118044889], [138.6375364888752, -34.9723118044889], [138.63731723494544, -34.9723118044889], [138.63720760798054, -34.9723118044889], [138.63720760798054, -34.97222197296049]]] ], - properties:[name:"SATFLB0001 - Control (Core)", externalId:"2", description:"SATFLB0001 - Control (Core)", notes:"Core monitoring plot some comment"]], + properties:[name:"SATFLB0001 - Control (Core)", externalId:"2", description:"SATFLB0001 - Control (Core)", notes:"Core monitoring plot some comment", activityType:'Core']], [ type:"Feature", geometry:[ @@ -212,7 +212,7 @@ class ParatooProtocolConfigSpec extends Specification { coordinates:[[[138.6371026907952, -34.971403261821905], [138.63709732396242, -34.972304399720215], [138.6381916652405, -34.972304399720215], [138.63819166764344, -34.9714076576406], [138.6371026907952, -34.971403261821905]]] ], properties:[ - name:"SATFLB0001 - Control (Fauna)", externalId:"2", description:"SATFLB0001 - Control (Fauna)", notes:"Fauna plot some comment" + name:"SATFLB0001 - Control (Fauna)", externalId:"2", description:"SATFLB0001 - Control (Fauna)", notes:"Fauna plot some comment", activityType:'Fauna' ] ] ] @@ -228,7 +228,8 @@ class ParatooProtocolConfigSpec extends Specification { name:"SATFLB0001 - Control (Core)", externalId:"2", description:"SATFLB0001 - Control (Core)", - notes:"Core monitoring plot some comment" + notes:"Core monitoring plot some comment", + activityType: 'Core' ] ], [ @@ -241,7 +242,8 @@ class ParatooProtocolConfigSpec extends Specification { name:"SATFLB0001 - Control (Fauna)", externalId:"2", description:"SATFLB0001 - Control (Fauna)", - notes:"Fauna plot some comment" + notes:"Fauna plot some comment", + activityType: 'Fauna' ] ] ] From 4c3ebedb67769129aaf3f7e41e42e37cb09e7488 Mon Sep 17 00:00:00 2001 From: temi Date: Fri, 31 May 2024 06:45:44 +1000 Subject: [PATCH 42/44] #965 - do not create record for "Other" species --- grails-app/conf/application.groovy | 2 +- .../au/org/ala/ecodata/ParatooService.groovy | 12 +++++++++--- .../au/org/ala/ecodata/ParatooServiceSpec.groovy | 12 ++++++++++++ 3 files changed, 22 insertions(+), 4 deletions(-) diff --git a/grails-app/conf/application.groovy b/grails-app/conf/application.groovy index da139f946..3c186a923 100644 --- a/grails-app/conf/application.groovy +++ b/grails-app/conf/application.groovy @@ -1490,4 +1490,4 @@ paratoo.defaultPlotLayoutViewModels = [ ] ] ] - +paratoo.species.specialCases = ["Other"] diff --git a/grails-app/services/au/org/ala/ecodata/ParatooService.groovy b/grails-app/services/au/org/ala/ecodata/ParatooService.groovy index d8b3077f2..15cd0dfd2 100644 --- a/grails-app/services/au/org/ala/ecodata/ParatooService.groovy +++ b/grails-app/services/au/org/ala/ecodata/ParatooService.groovy @@ -2097,9 +2097,15 @@ class ParatooService { } result.name = result.commonName ? result.scientificName ? "${result.scientificName} (${result.commonName})" : result.commonName : result.scientificName - - // record is only created if guid is present - result.guid = result.guid ?: Record.UNMATCHED_GUID + List specialCases = grailsApplication.config.getProperty("paratoo.species.specialCases", List) + // do not create record for special cases + if (specialCases.contains(name)) { + result.remove("guid") + } + else { + // record is only created if guid is present + result.guid = result.guid ?: Record.UNMATCHED_GUID + } result } } diff --git a/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy b/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy index 83f2c4f5c..046dade64 100644 --- a/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy +++ b/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy @@ -44,6 +44,7 @@ class ParatooServiceSpec extends MongoSpec implements ServiceUnitTest> null 1 * speciesReMatchService.searchByName(_, false, true) >> null + + when: // Do not create record when value equals special cases. Therefore, removes guid. + result = service.transformSpeciesName("Other") + outputSpeciesId = result.remove("outputSpeciesId") + + then: + outputSpeciesId != null + result == [name: "Other", scientificName: null, commonName: "Other", taxonRank: null] + 1 * speciesReMatchService.searchByName(_) >> null + 1 * speciesReMatchService.searchByName(_, false, true) >> null + } void "buildTreeFromParentChildRelationships should build tree correctly"() { From 50d4752e4c3af44283398dc29ced31a4086d36e9 Mon Sep 17 00:00:00 2001 From: temi Date: Fri, 31 May 2024 07:06:09 +1000 Subject: [PATCH 43/44] #965 - adding N/A --- grails-app/conf/application.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/grails-app/conf/application.groovy b/grails-app/conf/application.groovy index 3c186a923..1abd9646f 100644 --- a/grails-app/conf/application.groovy +++ b/grails-app/conf/application.groovy @@ -1490,4 +1490,4 @@ paratoo.defaultPlotLayoutViewModels = [ ] ] ] -paratoo.species.specialCases = ["Other"] +paratoo.species.specialCases = ["Other", "N/A"] From f6023f69c234b06019980ce78f038e00d7580204 Mon Sep 17 00:00:00 2001 From: chrisala Date: Fri, 31 May 2024 07:39:45 +1000 Subject: [PATCH 44/44] Fixed code review issue #942 --- grails-app/services/au/org/ala/ecodata/ParatooService.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/grails-app/services/au/org/ala/ecodata/ParatooService.groovy b/grails-app/services/au/org/ala/ecodata/ParatooService.groovy index 48d14c548..e03b704c8 100644 --- a/grails-app/services/au/org/ala/ecodata/ParatooService.groovy +++ b/grails-app/services/au/org/ala/ecodata/ParatooService.groovy @@ -875,7 +875,7 @@ class ParatooService { // The project/s for the site will be specified by a subsequent call to /projects siteData.projects = [] - Site site = Site.findByExternalId(ExternalId.IdType.MONITOR_PLOT_SELECTION_GUID, siteData.externalId) + Site site = Site.findByExternalId(ExternalId.IdType.MONITOR_PLOT_SELECTION_GUID, siteData.externalIds[0].externalId) Map result if (site) { result = siteService.update(siteData, site.siteId)