diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 3849705ae..f0ba78b6d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -8,6 +8,7 @@ on: - master - feature/** - hotfix/** + - release/** env: TZ: Australia/Canberra diff --git a/build.gradle b/build.gradle index 3eec3896b..56e413bff 100644 --- a/build.gradle +++ b/build.gradle @@ -20,7 +20,7 @@ plugins { id "com.gorylenko.gradle-git-properties" version "2.4.1" } -version "4.5.1" +version "4.6-SNAPSHOT" group "au.org.ala" description "Ecodata" diff --git a/grails-app/conf/application.groovy b/grails-app/conf/application.groovy index 3a77d5315..1abd9646f 100644 --- a/grails-app/conf/application.groovy +++ b/grails-app/conf/application.groovy @@ -501,6 +501,11 @@ app { if (!ala.baseURL) { ala.baseURL = "https://www.ala.org.au" } +bie.ws.url = "https://bie-ws.ala.org.au/" +bie.url = "https://bie.ala.org.au/" +namesmatching.url = "https://namematching-ws-test.ala.org.au/" +namematching.strategy = ["exactMatch", "vernacularMatch"] + if (!collectory.baseURL) { //collectory.baseURL = "https://collectory-dev.ala.org.au/" collectory.baseURL = "https://collections-test.ala.org.au/" @@ -1485,4 +1490,4 @@ paratoo.defaultPlotLayoutViewModels = [ ] ] ] - +paratoo.species.specialCases = ["Other", "N/A"] diff --git a/grails-app/controllers/au/org/ala/ecodata/AdminController.groovy b/grails-app/controllers/au/org/ala/ecodata/AdminController.groovy index 82fcd699a..a02d4aac7 100644 --- a/grails-app/controllers/au/org/ala/ecodata/AdminController.groovy +++ b/grails-app/controllers/au/org/ala/ecodata/AdminController.groovy @@ -737,7 +737,7 @@ class AdminController { def reSubmitDataSet() { String projectId = params.id String dataSetId = params.dataSetId - String userId = params.userId ?: userService.getCurrentUser().userId + String userId = params.userId ?: userService.currentUser()?.userId if (!projectId || !dataSetId || !userId) { render text: [message: "Bad request"] as JSON, status: HttpStatus.SC_BAD_REQUEST return diff --git a/grails-app/controllers/au/org/ala/ecodata/DataSetSummaryController.groovy b/grails-app/controllers/au/org/ala/ecodata/DataSetSummaryController.groovy new file mode 100644 index 000000000..7a5d1a292 --- /dev/null +++ b/grails-app/controllers/au/org/ala/ecodata/DataSetSummaryController.groovy @@ -0,0 +1,64 @@ +package au.org.ala.ecodata + +import org.apache.http.HttpStatus + +class DataSetSummaryController { + + static responseFormats = ['json', 'xml'] + static allowedMethods = [update:['POST', 'PUT'], delete:'DELETE', bulkUpdate: 'POST'] + + ProjectService projectService + + /** Updates a single dataset for a project */ + def update(String projectId) { + Map dataSet = request.JSON + projectId = projectId ?: dataSet.projectId + + if (!projectId) { + render status: HttpStatus.SC_BAD_REQUEST, text: "projectId is required" + return + } + + if (dataSet.projectId && dataSet.projectId != projectId) { + render status: HttpStatus.SC_BAD_REQUEST, text: "projectId must match the data set projectId" + return + } + + respond projectService.updateDataSet(projectId, dataSet) + } + + /** + * Updates multiple data sets for a project. + * This endpoint exists to support the use case of associating multiple data sets with a + * report and updating their publicationStatus when the report is submitted/approved. + * + * This method expects the projectId to be supplied via the URL and the data sets to be supplied in the request + * body as a JSON object with key="dataSets" and value=List of data sets. + */ + def bulkUpdate(String projectId) { + Map postBody = request.JSON + List dataSets = postBody?.dataSets + + if (!projectId) { + render status: HttpStatus.SC_BAD_REQUEST, text: "projectId is required" + return + } + + for (Map dataSet in dataSets) { + if (dataSet.projectId && dataSet.projectId != projectId) { + render status: HttpStatus.SC_BAD_REQUEST, text: "projectId must match the projectId in all supplied data sets" + return + } + } + + respond projectService.updateDataSets(projectId, dataSets) + } + + def delete(String projectId, String dataSetId) { + if (!projectId || !dataSetId) { + render status: HttpStatus.SC_BAD_REQUEST, text: "projectId and dataSetId are required" + return + } + respond projectService.deleteDataSet(projectId, dataSetId) + } +} diff --git a/grails-app/controllers/au/org/ala/ecodata/UrlMappings.groovy b/grails-app/controllers/au/org/ala/ecodata/UrlMappings.groovy index 7f85d0b43..c759f7a07 100644 --- a/grails-app/controllers/au/org/ala/ecodata/UrlMappings.groovy +++ b/grails-app/controllers/au/org/ala/ecodata/UrlMappings.groovy @@ -196,6 +196,12 @@ class UrlMappings { "/ws/project/getDefaultFacets"(controller: "project", action: "getDefaultFacets") "/ws/project/$projectId/dataSet/$dataSetId/records"(controller: "project", action: "fetchDataSetRecords") "/ws/admin/initiateSpeciesRematch"(controller: "admin", action: "initiateSpeciesRematch") + "/ws/dataSetSummary/$projectId/$dataSetId?"(controller :'dataSetSummary') { + + action = [POST:'update', PUT:'update', DELETE:'delete'] + } + + "/ws/dataSetSummary/bulkUpdate/$projectId"(controller:'dataSetSummary', action:'bulkUpdate') "/ws/document/download"(controller:"document", action:"download") diff --git a/grails-app/domain/au/org/ala/ecodata/ExternalId.groovy b/grails-app/domain/au/org/ala/ecodata/ExternalId.groovy index 3257e5320..8ab58aed6 100644 --- a/grails-app/domain/au/org/ala/ecodata/ExternalId.groovy +++ b/grails-app/domain/au/org/ala/ecodata/ExternalId.groovy @@ -13,7 +13,7 @@ class ExternalId implements Comparable { enum IdType { INTERNAL_ORDER_NUMBER, TECH_ONE_CODE, WORK_ORDER, GRANT_AWARD, GRANT_OPPORTUNITY, RELATED_PROJECT, MONITOR_PROTOCOL_INTERNAL_ID, MONITOR_PROTOCOL_GUID, TECH_ONE_CONTRACT_NUMBER, MONITOR_PLOT_GUID, - MONITOR_MINTED_COLLECTION_ID, UNSPECIFIED } + MONITOR_PLOT_SELECTION_GUID, MONITOR_MINTED_COLLECTION_ID, UNSPECIFIED } static constraints = { } diff --git a/grails-app/domain/au/org/ala/ecodata/Record.groovy b/grails-app/domain/au/org/ala/ecodata/Record.groovy index 0dfe00481..ddfd8682b 100644 --- a/grails-app/domain/au/org/ala/ecodata/Record.groovy +++ b/grails-app/domain/au/org/ala/ecodata/Record.groovy @@ -6,6 +6,8 @@ import org.bson.types.ObjectId class Record { // def grailsApplication + /** Represents a species guid that was unable to be matched against the ALA names list */ + static final String UNMATCHED_GUID = "A_GUID" static mapping = { occurrenceID index: true diff --git a/grails-app/domain/au/org/ala/ecodata/Site.groovy b/grails-app/domain/au/org/ala/ecodata/Site.groovy index 8b08eea50..a8fad1aec 100644 --- a/grails-app/domain/au/org/ala/ecodata/Site.groovy +++ b/grails-app/domain/au/org/ala/ecodata/Site.groovy @@ -181,4 +181,14 @@ class Site { status != Status.DELETED }.find() } + + static List findAllByExternalId(ExternalId.IdType idType, String externalId, Map params) { + where { + externalIds { + idType == idType + externalId == externalId + } + status != Status.DELETED + }.list(params) + } } diff --git a/grails-app/services/au/org/ala/ecodata/ParatooService.groovy b/grails-app/services/au/org/ala/ecodata/ParatooService.groovy index 7e2ee8013..cae51de9b 100644 --- a/grails-app/services/au/org/ala/ecodata/ParatooService.groovy +++ b/grails-app/services/au/org/ala/ecodata/ParatooService.groovy @@ -18,7 +18,7 @@ import static grails.async.Promises.task */ @Slf4j class ParatooService { - static final Object LOCK = new Object() + static final String DATASET_DATABASE_TABLE = 'Database Table' static final int PARATOO_MAX_RETRIES = 3 static final String PARATOO_PROTOCOL_PATH = '/protocols' @@ -77,6 +77,7 @@ class ParatooService { RecordService recordService MetadataService metadataService UserService userService + SpeciesReMatchService speciesReMatchService /** * The rules we use to find projects eligible for use by paratoo are: @@ -186,18 +187,7 @@ class ParatooService { dataSet.orgMintedIdentifier = paratooCollectionId.encodeAsOrgMintedIdentifier() log.info "Minting identifier for Monitor collection: ${paratooCollectionId}: ${dataSet.orgMintedIdentifier}" - Map result - synchronized (LOCK) { - Map latestProject = projectService.get(projectId) - if (!latestProject.custom) { - latestProject.custom = [:] - } - if (!latestProject.custom.dataSets) { - latestProject.custom.dataSets = [] - } - latestProject.custom.dataSets << dataSet - result = projectService.update([custom: latestProject.custom], projectId, false) - } + Map result = projectService.updateDataSet(projectId, dataSet) if (!result.error) { result.orgMintedIdentifier = dataSet.orgMintedIdentifier @@ -207,7 +197,7 @@ class ParatooService { private static String buildName(String protocolId, String displayDate, Project project) { ActivityForm protocolForm = ActivityForm.findByExternalId(protocolId) - String dataSetName = protocolForm?.name + " - " + displayDate + " (" + project.name + ")" + String dataSetName = protocolForm?.name + " - " + displayDate dataSetName } @@ -232,20 +222,20 @@ class ParatooService { Map authHeader = getAuthHeader() Promise promise = task { + userService.setCurrentUser(userId) asyncFetchCollection(collection, authHeader, userId, project) } promise.onError { Throwable e -> log.error("An error occurred feching ${collection.orgMintedUUID}: ${e.message}", e) + userService.clearCurrentUser() } - def result - synchronized (LOCK) { - Map latestProject = projectService.get(project.id) - Map latestDataSet = latestProject.custom?.dataSets?.find { it.dataSetId == collection.orgMintedUUID } - latestDataSet.putAll(dataSet) - result = projectService.update([custom: latestProject.custom], project.id, false) + promise.onComplete { Map result -> + userService.clearCurrentUser() } + def result = projectService.updateDataSet(project.id, dataSet) + [updateResult: result, promise: promise] } @@ -288,8 +278,16 @@ class ParatooService { surveyDataAndObservations = recursivelyTransformData(form.sections[0].template.dataModel, surveyDataAndObservations, form.name, 1, config) // If we are unable to create a site, null will be returned - assigning a null siteId is valid. + String siteName = null if (!dataSet.siteId) { - dataSet.siteId = createSiteFromSurveyData(surveyDataAndObservations, collection, surveyId, project.project, config, form) + try { + Map site = createSiteFromSurveyData(surveyDataAndObservations, collection, surveyId, project.project, config, form) + dataSet.siteId = site.siteId + siteName = site.name + } + catch (Exception ex) { + log.error("Error creating site for ${collection.orgMintedUUID}: ${ex.message}") + } } // plot layout is of type geoMap. Therefore, expects a site id. @@ -297,6 +295,14 @@ class ParatooService { surveyDataAndObservations[PARATOO_DATAMODEL_PLOT_LAYOUT] = dataSet.siteId } + dataSet.startDate = config.getStartDate(surveyDataAndObservations) + dataSet.endDate = config.getEndDate(surveyDataAndObservations) + dataSet.format = DATASET_DATABASE_TABLE + dataSet.sizeUnknown = true + // Update the data set name as the information supplied during /mint-identifier isn't enough + // to ensure uniqueness. + dataSet.name = buildUpdatedDataSetSummaryName(siteName, dataSet.startDate, dataSet.endDate, form.name, surveyId, config) + // Delete previously created activity so that duplicate species records are not created. // Updating existing activity will also create duplicates since it relies on outputSpeciesId to determine // if a record is new and new ones are created by code. @@ -304,26 +310,34 @@ class ParatooService { activityService.delete(dataSet.activityId, true) } - String activityId = createActivityFromSurveyData(form, surveyDataAndObservations, surveyId, dataSet.siteId, userId) + String activityId = createActivityFromSurveyData(form, surveyDataAndObservations, surveyId, dataSet, userId) List records = recordService.getAllByActivity(activityId) dataSet.areSpeciesRecorded = records?.size() > 0 dataSet.activityId = activityId - dataSet.startDate = config.getStartDate(surveyDataAndObservations) - dataSet.endDate = config.getEndDate(surveyDataAndObservations) - dataSet.format = DATASET_DATABASE_TABLE - dataSet.sizeUnknown = true - - synchronized (LOCK) { - Map latestProject = projectService.get(project.project.projectId) - Map latestDataSet = latestProject.custom?.dataSets?.find { it.dataSetId == collection.orgMintedUUID } - latestDataSet.putAll(dataSet) - projectService.update([custom: latestProject.custom], project.id, false) - } + projectService.updateDataSet(project.id, dataSet) } } } + protected static String buildUpdatedDataSetSummaryName(String siteName, String startDate, String endDate, String protocolName, ParatooCollectionId surveyId, ParatooProtocolConfig config) { + String name = protocolName + //The site name for non-plot based data sets is not used as it contains the same data (protocol and time) as the name + if (siteName && config.usesPlotLayout) { + name += " (" + siteName + ")" + } + if (startDate && endDate && startDate != endDate) { + name += " - " + DateUtil.formatAsDisplayDateTime(startDate) + " to " + DateUtil.formatAsDisplayDateTime(endDate) + } + else if (startDate) { + name += " - " +DateUtil.formatAsDisplayDateTime(startDate) + } + else { + name += " - " + DateUtil.formatAsDisplayDateTime(surveyId.eventTime) + } + name + } + /** * Rearrange survey data to match the data model. * e.g. [a: [b: [c: 1, d: 2], d: 1], b: [c: 1, d: 2]] => [b: [c: 1, d: 2, a: [d: 1]]] @@ -431,10 +445,13 @@ class ParatooService { private boolean protocolCheck(String userId, String projectId, String protocolId, boolean read) { List projects = userProjects(userId) + ParatooProject project = projects.find { it.id == projectId } - boolean protocol = project?.protocols?.find { it.externalIds.find { it.externalId == protocolId } } - int minimumAccess = read ? AccessLevel.projectParticipant.code : AccessLevel.editor.code - protocol && project.accessLevel.code >= minimumAccess + ActivityForm protocol = project?.protocols?.find { it.externalIds.find { it.externalId == protocolId } } + int minAccessLevel = AccessLevel.projectParticipant.code + // Note we don't need to include a check for ADMIN_ONLY_PROTOCOLS here as those protocol will have already be filtered + // out of the list of protocols attached to the project in findProjectProtocols if the user isn't an admin. + protocol && project.accessLevel.code >= minAccessLevel } Map findDataSet(String userId, String orgMintedUUID) { @@ -456,14 +473,20 @@ class ParatooService { * @param siteId * @return */ - private String createActivityFromSurveyData(ActivityForm activityForm, Map surveyObservations, ParatooCollectionId collection, String siteId, String userId) { + private String createActivityFromSurveyData(ActivityForm activityForm, Map surveyObservations, ParatooCollectionId collection, Map dataSet, String userId) { Map activityProps = [ type : activityForm.name, formVersion : activityForm.formVersion, description : "Activity submitted by monitor", projectId : collection.projectId, - publicationStatus: "published", - siteId : siteId, + publicationStatus: PublicationStatus.PUBLISHED, + siteId : dataSet.siteId, + startDate : dataSet.startDate, + endDate : dataSet.endDate, + plannedStartDate : dataSet.startDate, + plannedEndDate : dataSet.endDate, + progress : Activity.FINISHED, + externalIds : [new ExternalId(idType: ExternalId.IdType.MONITOR_MINTED_COLLECTION_ID, externalId: dataSet.dataSetId)], userId : userId, outputs : [[ data: surveyObservations, @@ -523,35 +546,39 @@ class ParatooService { // used by protocols like bird survey where a point represents a sight a bird has been observed in a // bird survey plot def location = output[model.name] - if (location instanceof Map) { - output[model.name] = [ - type : 'Feature', - geometry : [ - type : 'Point', - coordinates: [location.lng, location.lat] - ], - properties: [ - name : "Point ${formName}-${featureId}", - externalId: location.id, - id: "${formName}-${featureId}" - ] - ] - } - else if (location instanceof List) { - String name - switch (config?.geometryType) { - case "LineString": - name = "LineString ${formName}-${featureId}" - output[model.name] = ParatooProtocolConfig.createLineStringFeatureFromGeoJSON (location, name, null, name) - break - default: - name = "Polygon ${formName}-${featureId}" - output[model.name] = ParatooProtocolConfig.createFeatureFromGeoJSON (location, name, null, name) - break + if (location) { + if (location instanceof Map) { + output[model.name] = [ + type : 'Feature', + geometry : [ + type : 'Point', + coordinates: [location.lng, location.lat] + ], + properties: [ + name : "Point ${formName}-${featureId}", + externalId: location.id, + id : "${formName}-${featureId}" + ] + ] + } else if (location instanceof List) { + String name + switch (config?.geometryType) { + case "LineString": + name = "LineString ${formName}-${featureId}" + output[model.name] = ParatooProtocolConfig.createLineStringFeatureFromGeoJSON(location, name, null, name) + break + default: + name = "Polygon ${formName}-${featureId}" + output[model.name] = ParatooProtocolConfig.createFeatureFromGeoJSON(location, name, null, name) + break + } } - } - featureId ++ + featureId ++ + } + else { + output[model.name] = null + } break case "image": case "document": @@ -566,44 +593,80 @@ class ParatooService { output } - private String createSiteFromSurveyData(Map observation, ParatooCollection collection, ParatooCollectionId surveyId, Project project, ParatooProtocolConfig config, ActivityForm form) { + private Map createSiteFromSurveyData(Map observation, ParatooCollection collection, ParatooCollectionId surveyId, Project project, ParatooProtocolConfig config, ActivityForm form) { String siteId = null + Date updatedPlotLayoutDate // Create a site representing the location of the collection + Map siteProps = null Map geoJson = config.getGeoJson(observation, form) if (geoJson) { - Map siteProps = siteService.propertiesFromGeoJson(geoJson, 'upload') + siteProps = siteService.propertiesFromGeoJson(geoJson, 'upload') List features = geoJson?.features ?: [] geoJson.remove('features') siteProps.features = features - siteProps.type = Site.TYPE_SURVEY_AREA + if (features) + siteProps.type = Site.TYPE_COMPOUND + else + siteProps.type = Site.TYPE_SURVEY_AREA siteProps.publicationStatus = PublicationStatus.PUBLISHED siteProps.projects = [project.projectId] String externalId = geoJson.properties?.externalId - if (externalId) { - siteProps.externalIds = [new ExternalId(idType: ExternalId.IdType.MONITOR_PLOT_GUID, externalId: externalId)] + if (config.usesPlotLayout) { + if (externalId) { + siteProps.externalIds = [new ExternalId(idType: ExternalId.IdType.MONITOR_PLOT_GUID, externalId: externalId)] + } + else { + log.error("No externalId found for plot layout for survey ${collection.orgMintedUUID}, project ${project.projectId}") + } + } + else { + // non-plot based data sets will have the dataSetId/orgMintedUUID as the external id + siteProps.externalIds = [new ExternalId(idType: ExternalId.IdType.MONITOR_PLOT_GUID, externalId: collection.orgMintedUUID)] } Site site // create new site for every non-plot submission if (config.usesPlotLayout) { - site = Site.findByExternalId(ExternalId.IdType.MONITOR_PLOT_GUID, externalId) - if (site?.features) { - siteProps.features?.addAll(site.features) - } + updatedPlotLayoutDate = config.getPlotLayoutUpdatedAt(observation) + List sites = Site.findAllByExternalId(ExternalId.IdType.MONITOR_PLOT_GUID, externalId, [sort: "lastUpdated", order: "desc"]) + if (sites) + site = sites.first() } Map result + // If the plot layout has been updated, create a new site if (!site) { result = siteService.create(siteProps) - } else { + } + else if(isUpdatedPlotLayout(site.lastUpdated, updatedPlotLayoutDate)){ + siteProps.name = "${siteProps.name} - ${DateUtil.formatAsDisplayDateTime(updatedPlotLayoutDate)}" + result = siteService.create(siteProps) + } + else { result = [siteId: site.siteId] } + if (result.error) { // Don't treat this as a fatal error for the purposes of responding to the paratoo request log.error("Error creating a site for survey " + collection.orgMintedUUID + ", project " + project.projectId + ": " + result.error) } siteId = result.siteId } - siteId + [siteId:siteId, name:siteProps?.name] + } + + /** + * check if the plot layout has been updated after site has been updated. This means user has edited plot layout and + * a new site should be created. + * @param siteLastUpdated + * @param plotLayoutLastUpdated + * @return + */ + static boolean isUpdatedPlotLayout (Date siteLastUpdated, Date plotLayoutLastUpdated) { + if ((siteLastUpdated != null) && (plotLayoutLastUpdated != null)) { + return plotLayoutLastUpdated.after(siteLastUpdated) + } + + return false } private Map syncParatooProtocols(List protocols) { @@ -752,7 +815,7 @@ class ParatooService { // Monitor has users selecting a point as an approximate survey location then // laying out the plot using GPS when at the site. We only want to return the approximate planning // sites from this call - List plotSelections = sites.findAll{it.type == Site.TYPE_SURVEY_AREA && it.extent?.geometry?.type == 'Point'} + List plotSelections = sites.findAll{it.externalIds?.find{externalId -> externalId.idType == ExternalId.IdType.MONITOR_PLOT_SELECTION_GUID}} Map attributes = [ id:project.projectId, @@ -785,10 +848,6 @@ class ParatooService { dataSet } - private static String buildSurveyQueryString(int start, int limit, String createdAt) { - "?populate=deep&sort=updatedAt&pagination[start]=$start&pagination[limit]=$limit&filters[createdAt][\$eq]=$createdAt" - } - Map retrieveSurveyAndObservations(ParatooCollection collection, Map authHeader = null) { String apiEndpoint = PARATOO_DATA_PATH Map payload = [ @@ -817,7 +876,7 @@ class ParatooService { // The project/s for the site will be specified by a subsequent call to /projects siteData.projects = [] - Site site = Site.findByExternalId(ExternalId.IdType.MONITOR_PLOT_GUID, siteData.externalId) + Site site = Site.findByExternalId(ExternalId.IdType.MONITOR_PLOT_SELECTION_GUID, siteData.externalIds[0].externalId) Map result if (site) { result = siteService.update(siteData, site.siteId) @@ -834,7 +893,7 @@ class ParatooService { site.projects = [] // get all projects for the user I suppose - not sure why this isn't in the payload as it's in the UI... site.type = Site.TYPE_SURVEY_AREA - site.externalIds = [new ExternalId(idType: ExternalId.IdType.MONITOR_PLOT_GUID, externalId: geoJson.properties.externalId)] + site.externalIds = [new ExternalId(idType: ExternalId.IdType.MONITOR_PLOT_SELECTION_GUID, externalId: geoJson.properties.externalId)] site.publicationStatus = PublicationStatus.PUBLISHED // Mark the plot as read only as it is managed by the Monitor app @@ -861,7 +920,7 @@ class ParatooService { siteExternalIds.each { String siteExternalId -> - Site site = Site.findByExternalId(ExternalId.IdType.MONITOR_PLOT_GUID, siteExternalId) + Site site = Site.findByExternalId(ExternalId.IdType.MONITOR_PLOT_SELECTION_GUID, siteExternalId) if (site) { site.projects = site.projects ?: [] if (!site.projects.contains(project.id)) { @@ -1184,8 +1243,9 @@ class ParatooService { ArrayDeque modelVisitStack = new ArrayDeque<>() documentation = deepCopy(documentation) Map components = deepCopy(getComponents(documentation)) + boolean record = config.createSpeciesRecord - Map template = [dataModel: [], viewModel: [], modelName: capitalizeModelName(protocol.attributes.name), record: true, relationships: [ecodata: [:], apiOutput: [:]]] + Map template = [dataModel: [], viewModel: [], modelName: capitalizeModelName(protocol.attributes.name), record: record, relationships: [ecodata: [:], apiOutput: [:]]] Map properties = deepCopy(findProtocolEndpointDefinition(protocol, documentation)) if (properties == null) { throw new NotFoundException("No protocol endpoint found for ${protocol.attributes.endpointPrefix}/bulk") @@ -2009,7 +2069,7 @@ class ParatooService { /** * Transforms a species name to species object used by ecodata. * e.g. Acacia glauca [Species] (scientific: Acacia glauca Willd.) - * [name: "Acacia glauca Willd.", scientificName: "Acacia glauca Willd.", guid: "A_GUID"] + * [name: "Acacia glauca Willd.", scientificName: "Acacia glauca Willd.", commonName: "Acacia glauca", guid: "A_GUID"] * Guid is necessary to generate species occurrence record. Guid is found by searching the species name with BIE. If not found, then a default value is added. * @param name * @return @@ -2020,30 +2080,42 @@ class ParatooService { } String regex = "([^\\[\\(]*)(?:\\[(.*)\\])?\\s*(?:\\(scientific:\\s*(.*?)\\))?" + String commonName, scientificName = name Pattern pattern = Pattern.compile(regex) Matcher matcher = pattern.matcher(name) - Map result = [name: name, scientificName: name, commonName: name, outputSpeciesId: UUID.randomUUID().toString()] + Map result = [scientificName: name, commonName: name, outputSpeciesId: UUID.randomUUID().toString()] if (matcher.find()) { - String commonName = matcher.group(1)?.trim() - String scientificName = matcher.group(3)?.trim() - result.commonName = commonName ?: result.commonName + commonName = matcher.group(1)?.trim() + scientificName = matcher.group(3)?.trim() result.taxonRank = matcher.group(2)?.trim() - result.scientificName = scientificName ?: commonName ?: result.scientificName - result.name = scientificName ?: commonName ?: result.name + result.scientificName = scientificName + result.commonName = commonName } - metadataService.autoPopulateSpeciesData(result) + Map resp = speciesReMatchService.searchByName(scientificName) + if (resp) { + result.putAll(resp) + } // try again with common name - if ((result.guid == null) && result.commonName) { - def speciesObject = [scientificName: result.commonName] - metadataService.autoPopulateSpeciesData(speciesObject) - result.guid = speciesObject.guid - result.scientificName = result.scientificName ?: speciesObject.scientificName + if ((result.guid == null) && commonName) { + resp = speciesReMatchService.searchByName(commonName, false, true) + if (resp) { + result.putAll(resp) + result.commonName = commonName + } } - // record is only created if guid is present - result.guid = result.guid ?: "A_GUID" + result.name = result.commonName ? result.scientificName ? "${result.scientificName} (${result.commonName})" : result.commonName : result.scientificName + List specialCases = grailsApplication.config.getProperty("paratoo.species.specialCases", List) + // do not create record for special cases + if (specialCases.contains(name)) { + result.remove("guid") + } + else { + // record is only created if guid is present + result.guid = result.guid ?: Record.UNMATCHED_GUID + } result } } diff --git a/grails-app/services/au/org/ala/ecodata/ProjectService.groovy b/grails-app/services/au/org/ala/ecodata/ProjectService.groovy index b1ee890db..103cbb4e5 100644 --- a/grails-app/services/au/org/ala/ecodata/ProjectService.groovy +++ b/grails-app/services/au/org/ala/ecodata/ProjectService.groovy @@ -26,6 +26,11 @@ class ProjectService { static final ENHANCED = 'enhanced' static final PRIVATE_SITES_REMOVED = 'privatesitesremoved' + /** A Map containing a per-project lock for synchronizing locks for updates. The purpose of this + * is to support concurrent edits on different project data set summaries which are currently modelled as + * an embedded array but can be added and updated by both the UI and the Monitor (Parataoo) application API */ + static final Map PROJECT_UPDATE_LOCKS = Collections.synchronizedMap([:].withDefault{ new Object() }) + GrailsApplication grailsApplication MessageSource messageSource SessionLocaleResolver localeResolver @@ -51,6 +56,8 @@ class ProjectService { grailsApplication.mainContext.commonService }*/ + + def getBrief(listOfIds, version = null) { if (listOfIds) { if (version) { @@ -469,30 +476,37 @@ class ProjectService { } def update(Map props, String id, Boolean shouldUpdateCollectory = true) { - Project project = Project.findByProjectId(id) - if (project) { - // retrieve any project activities associated with the project - List projectActivities = projectActivityService.getAllByProject(id) - props = includeProjectFundings(props) - props = includeProjectActivities(props, projectActivities) + synchronized (PROJECT_UPDATE_LOCKS.get(id)) { + Project project = Project.findByProjectId(id) + if (project) { + // retrieve any project activities associated with the project + List projectActivities = projectActivityService.getAllByProject(id) + props = includeProjectFundings(props) + props = includeProjectActivities(props, projectActivities) - try { - bindEmbeddedProperties(project, props) - commonService.updateProperties(project, props) - if (shouldUpdateCollectory) { - updateCollectoryLinkForProject(project, props) + try { + // Custom currently holds keys "details" and "dataSets". Only update the "custom" properties + // that are supplied in the update, leaving the others intact. + if (project.custom && props.custom) { + project.custom.putAll(props.remove('custom')) + } + bindEmbeddedProperties(project, props) + commonService.updateProperties(project, props) + if (shouldUpdateCollectory) { + updateCollectoryLinkForProject(project, props) + } + return [status: 'ok'] + } catch (Exception e) { + Project.withSession { session -> session.clear() } + def error = "Error updating project ${id} - ${e.message}" + log.error error, e + return [status: 'error', error: error] } - return [status: 'ok'] - } catch (Exception e) { - Project.withSession { session -> session.clear() } - def error = "Error updating project ${id} - ${e.message}" - log.error error, e + } else { + def error = "Error updating project - no such id ${id}" + log.error error return [status: 'error', error: error] } - } else { - def error = "Error updating project - no such id ${id}" - log.error error - return [status: 'error', error: error] } } @@ -1054,4 +1068,77 @@ class ProjectService { records } + /** + * Updates a single data set associated with a project. Because the datasets are stored as an embedded + * array in the Project collection, this method is synchronized on the project to avoid concurrent updates to + * different data sets overwriting each other. + * Due to the way it's been modelled as an embedded array, the client is allowed to supply a dataSetId + * when creating a new data set (e.g. a data set created by a submission from the Monitor app uses the + * submissionId as the dataSetId). + * @param projectId The project to update + * @param dataSet the data set to update. + * @return + */ + Map updateDataSet(String projectId, Map dataSet) { + updateDataSets(projectId, [dataSet]) + } + + /** + * Updates multiple data sets associated with a project at the same time. This method exists to support + * the use case of associating multiple data sets with a report and updating their publicationStatus when + * the report is submitted/approved. + * + * Because the datasets are stored as an embedded + * array in the Project collection, this method is synchronized on the project to avoid concurrent updates to + * different data sets overwriting each other. + * Due to the way it's been modelled as an embedded array, the client is allowed to supply a dataSetId + * when creating a new data set (e.g. a data set created by a submission from the Monitor app uses the + * submissionId as the dataSetId). + * @param projectId The project to update + * @param dataSet the data sets to update. + * @return + */ + Map updateDataSets(String projectId, List dataSets) { + synchronized (PROJECT_UPDATE_LOCKS.get(projectId)) { + Project.withNewSession { // Ensure that the queried Project is not cached in the current session which can cause stale data + Project project = Project.findByProjectId(projectId) + if (!project) { + return [status: 'error', error: "No project exists with projectId=${projectId}"] + } + for (Map dataSet in dataSets) { + if (!dataSet.dataSetId) { + dataSet.dataSetId = Identifiers.getNew(true, '') + } + Map matchingDataSet = project.custom?.dataSets?.find { it.dataSetId == dataSet.dataSetId } + if (matchingDataSet) { + matchingDataSet.putAll(dataSet) + } else { + if (!project.custom) { + project.custom = [:] + } + if (!project.custom?.dataSets) { + project.custom.dataSets = [] + } + project.custom.dataSets.add(dataSet) + } + } + update([custom: project.custom], project.projectId, false) + } + } + } + + Map deleteDataSet(String projectId, String dataSetId) { + synchronized (PROJECT_UPDATE_LOCKS.get(projectId)) { + Project project = Project.findByProjectId(projectId) + + boolean foundMatchingDataSet = project?.custom?.dataSets?.removeAll { it.dataSetId == dataSetId } + if (!foundMatchingDataSet) { + return [status: 'error', error: 'No such data set'] + } + else { + update([custom: project.custom], project.projectId, false) + } + } + } + } \ No newline at end of file diff --git a/grails-app/services/au/org/ala/ecodata/SiteService.groovy b/grails-app/services/au/org/ala/ecodata/SiteService.groovy index 4e5c746bc..37db8dbe7 100644 --- a/grails-app/services/au/org/ala/ecodata/SiteService.groovy +++ b/grails-app/services/au/org/ala/ecodata/SiteService.groovy @@ -623,7 +623,19 @@ class SiteService { def populateLocationMetadataForSite(Map site) { - def siteGeom = geometryAsGeoJson(site) + Map siteGeom + if (site.type == Site.TYPE_COMPOUND) { + siteGeom = [ + type:'GeometryCollection', + geometries: [ + site.features.collect{it.geometry} + ] + ] + } + else { + siteGeom = geometryAsGeoJson(site) + } + if (siteGeom) { GeometryJSON gjson = new GeometryJSON() Geometry geom = gjson.read((siteGeom as JSON).toString()) diff --git a/grails-app/services/au/org/ala/ecodata/SpeciesReMatchService.groovy b/grails-app/services/au/org/ala/ecodata/SpeciesReMatchService.groovy index e074cd730..d02361bc5 100644 --- a/grails-app/services/au/org/ala/ecodata/SpeciesReMatchService.groovy +++ b/grails-app/services/au/org/ala/ecodata/SpeciesReMatchService.groovy @@ -85,9 +85,60 @@ class SpeciesReMatchService { name = name?.toLowerCase() ?: "" cacheService.get('bie-search-auto-' + name, { def encodedQuery = URLEncoder.encode(name ?: '', "UTF-8") - def url = "${grailsApplication.config.getProperty('bie.url')}ws/search/auto.jsonp?q=${encodedQuery}&limit=${limit}&idxType=TAXON" + def url = "${grailsApplication.config.getProperty('bie.ws.url')}ws/search/auto.jsonp?q=${encodedQuery}&limit=${limit}&idxType=TAXON" webService.getJson(url) }) } + + Map searchByName (String name, boolean addDetails = false, boolean useVernacularSearch = false ) { + Map result + if (!useVernacularSearch) + result = searchNameMatchingServer(name) + else + result = searchByVernacularNameOnNameMatchingServer(name) + List strategy = grailsApplication.config.getProperty('namematching.strategy', List) + if (strategy.contains(result?.matchType)) { + Map resp = [ + scientificName: result.scientificName, + commonName: result.vernacularName, + guid: result.taxonConceptID, + taxonRank: result.rank + ] + + if(addDetails) { + resp.put('details', result) + } + + return resp + } + } + + Map searchNameMatchingServer(String name) { + name = name?.toLowerCase() ?: "" + cacheService.get('name-matching-server-' + name, { + def encodedQuery = URLEncoder.encode(name ?: '', "UTF-8") + def url = "${grailsApplication.config.getProperty('namesmatching.url')}api/search?q=${encodedQuery}" + def resp = webService.getJson(url) + if (!resp.success) { + return null + } + + resp + }) + } + + Map searchByVernacularNameOnNameMatchingServer (String name) { + name = name?.toLowerCase() ?: "" + cacheService.get('name-matching-server-vernacular-name' + name, { + def encodedQuery = URLEncoder.encode(name ?: '', "UTF-8") + def url = "${grailsApplication.config.getProperty('namesmatching.url')}api/searchByVernacularName?vernacularName=${encodedQuery}" + def resp = webService.getJson(url) + if (!resp.success) { + return null + } + + resp + }) + } } diff --git a/src/main/groovy/au/org/ala/ecodata/DateUtil.groovy b/src/main/groovy/au/org/ala/ecodata/DateUtil.groovy index 8c5831423..ee5d8a0ae 100644 --- a/src/main/groovy/au/org/ala/ecodata/DateUtil.groovy +++ b/src/main/groovy/au/org/ala/ecodata/DateUtil.groovy @@ -54,6 +54,11 @@ class DateUtil { dateTime.format(DISPLAY_DATE_TIME_FORMATTER) } + static String formatAsDisplayDateTime(String isoDateString) { + Date date = parse(isoDateString) + formatAsDisplayDateTime(date) + } + /** * Returns a formatted string representing the financial year a report or activity falls into, based on * the end date. This method won't necessarily work for start dates as it will subtract a day from the value diff --git a/src/main/groovy/au/org/ala/ecodata/converter/FeatureConverter.groovy b/src/main/groovy/au/org/ala/ecodata/converter/FeatureConverter.groovy index 03165ef20..3a4cae88a 100644 --- a/src/main/groovy/au/org/ala/ecodata/converter/FeatureConverter.groovy +++ b/src/main/groovy/au/org/ala/ecodata/converter/FeatureConverter.groovy @@ -4,27 +4,27 @@ class FeatureConverter implements RecordFieldConverter { List convert(Map data, Map metadata = [:]) { Map record = [:] + if (data[metadata.name]) { + Double latitude = getDecimalLatitude(data[metadata.name]) + Double longitude = getDecimalLongitude(data[metadata.name]) + // Don't override decimalLongitud or decimalLatitude in case they are null, site info could've already set them + if (latitude != null) { + record.decimalLatitude = latitude + } - Double latitude = getDecimalLatitude(data[metadata.name]) - Double longitude = getDecimalLongitude(data[metadata.name]) - - // Don't override decimalLongitud or decimalLatitude in case they are null, site info could've already set them - if(latitude != null) { - record.decimalLatitude = latitude - } - - if (longitude != null) { - record.decimalLongitude = longitude - } + if (longitude != null) { + record.decimalLongitude = longitude + } - Map dwcMappings = extractDwcMapping(metadata) + Map dwcMappings = extractDwcMapping(metadata) - record << getDwcAttributes(data, dwcMappings) + record << getDwcAttributes(data, dwcMappings) - if (data.dwcAttribute) { - record[data.dwcAttribute] = data.value + if (data.dwcAttribute) { + record[data.dwcAttribute] = data.value + } } [record] diff --git a/src/main/groovy/au/org/ala/ecodata/metadata/SpeciesUrlGetter.groovy b/src/main/groovy/au/org/ala/ecodata/metadata/SpeciesUrlGetter.groovy new file mode 100644 index 000000000..89a7dffc1 --- /dev/null +++ b/src/main/groovy/au/org/ala/ecodata/metadata/SpeciesUrlGetter.groovy @@ -0,0 +1,26 @@ +package au.org.ala.ecodata.metadata + +import pl.touk.excel.export.getters.Getter +import au.org.ala.ecodata.Record + +class SpeciesUrlGetter extends OutputDataGetter implements Getter { + String biePrefix + SpeciesUrlGetter(String propertyName, Map dataNode, Map documentMap, TimeZone timeZone, String biePrefix) { + super(propertyName, dataNode, documentMap, timeZone) + this.biePrefix = biePrefix + } + + @Override + def species(Object node, Value outputValue) { + def val = outputValue.value + if (!val?.name) { + return "" + } + if (!val?.guid || val.guid == Record.UNMATCHED_GUID) { + return "Unmatched name" + } + return biePrefix+val.guid + } + + +} diff --git a/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy b/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy index e6b0be669..b8428020c 100644 --- a/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy +++ b/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy @@ -1,7 +1,6 @@ package au.org.ala.ecodata.paratoo import au.org.ala.ecodata.* -import au.org.ala.ecodata.converter.ISODateBindingConverter import au.org.ala.ecodata.metadata.OutputMetadata import au.org.ala.ecodata.metadata.PropertyAccessor import com.fasterxml.jackson.annotation.JsonIgnoreProperties @@ -13,10 +12,14 @@ import org.locationtech.jts.geom.Geometry @Slf4j @JsonIgnoreProperties(['metaClass', 'errors', 'expandoMetaClass']) class ParatooProtocolConfig { - + static final String FAUNA_PLOT = 'Fauna plot' + static final String CORE_PLOT = 'Core monitoring plot' + static final String FAUNA_PLOT_SHORT = 'Fauna' + static final String CORE_PLOT_SHORT = 'Core' String name String apiEndpoint boolean usesPlotLayout = true + boolean createSpeciesRecord = true List tags String geometryType = 'Polygon' @@ -25,12 +28,11 @@ class ParatooProtocolConfig { String endDatePath = 'end_date_time' String surveyIdPath = 'survey_metadata' String plotVisitPath = 'plot_visit' - String plotProtocolObservationDatePath = "date_time" - String plotVisitStartDatePath = "${plotVisitPath}.start_date" - String plotVisitEndDatePath = "${plotVisitPath}.end_date" String plotLayoutPath = "${plotVisitPath}.plot_layout" String plotLayoutIdPath = "${plotLayoutPath}.id" + String plotLayoutUpdatedAtPath = "${plotLayoutPath}.updatedAt" String plotLayoutPointsPath = "${plotLayoutPath}.plot_points" + String faunaPlotPointPath = "${plotLayoutPath}.fauna_plot_point" String plotSelectionPath = "${plotLayoutPath}.plot_selection" String plotLayoutDimensionLabelPath = "${plotLayoutPath}.plot_dimensions.label" String plotLayoutTypeLabelPath = "${plotLayoutPath}.plot_type.label" @@ -55,46 +57,13 @@ class ParatooProtocolConfig { return null } - def date - if (usesPlotLayout) { - List dates = getDatesFromObservation(surveyData) - date = dates ? DateUtil.format(dates.first()) : null - return date + def date = getProperty(surveyData, startDatePath) + if (!date) { + date = getPropertyFromSurvey(surveyData, startDatePath) } - else { - date = getProperty(surveyData, startDatePath) - if (!date) { - date = getPropertyFromSurvey(surveyData, startDatePath) - } - date = getFirst(date) - return removeMilliseconds(date) - } - } - - /** - * Get date from plotProtocolObservationDatePath and sort them. - * @param surveyData - reverse lookup output which includes survey and observation data - * @return - */ - List getDatesFromObservation(Map surveyData) { - Map surveysData = surveyData.findAll { key, value -> - ![ getSurveyAttributeName(), ParatooService.PARATOO_DATAMODEL_PLOT_SELECTION, - ParatooService.PARATOO_DATAMODEL_PLOT_VISIT, ParatooService.PARATOO_DATAMODEL_PLOT_LAYOUT].contains(key) - } - List result = [] - ISODateBindingConverter converter = new ISODateBindingConverter() - surveysData.each { key, value -> - def dates = getProperty(value, plotProtocolObservationDatePath) - dates = dates instanceof List ? dates : [dates] - - result.addAll(dates.collect { String date -> - date ? converter.convert(date, ISODateBindingConverter.FORMAT) : null - }) - } - - result = result.findAll { it != null } - result.sort() + date = getFirst(date) + return removeMilliseconds(date) } def getPropertyFromSurvey(Map surveyData, String path) { @@ -107,21 +76,23 @@ class ParatooProtocolConfig { return null } - def date - if (usesPlotLayout) { - def dates = getDatesFromObservation(surveyData) - date = dates ? DateUtil.format(dates.last()) : null - return date + def date = getProperty(surveyData, endDatePath) + if (!date) { + date = getPropertyFromSurvey(surveyData, endDatePath) } - else { - date = getProperty(surveyData, endDatePath) - if (!date) { - date = getPropertyFromSurvey(surveyData, endDatePath) - } - date = getFirst(date) - return removeMilliseconds(date) + date = getFirst(date) + return removeMilliseconds(date) + } + + Date getPlotLayoutUpdatedAt(Map surveyData) { + def date = getProperty(surveyData, plotLayoutUpdatedAtPath) + if (!date) { + date = getPropertyFromSurvey(surveyData, plotLayoutUpdatedAtPath) } + + date = getFirst(date) + date ? DateUtil.parseWithMilliseconds(date) : null } Map getSurveyId(Map surveyData) { @@ -180,7 +151,8 @@ class ParatooProtocolConfig { List features = [] paths.each { String name, node -> if (node instanceof Boolean) { - features.add(output[name]) + if (output[name]) + features.add(output[name]) // todo later: add featureIds and modelId for compliance with feature behaviour of reports } @@ -246,10 +218,6 @@ class ParatooProtocolConfig { Map geoJson = null if (usesPlotLayout) { geoJson = extractSiteDataFromPlotVisit(output) - // get list of all features associated with observation - if (geoJson && form && output) { - geoJson.features = extractFeatures(output, form) - } } else if (geometryPath) { geoJson = extractSiteDataFromPath(output) @@ -257,20 +225,10 @@ class ParatooProtocolConfig { else if (form && output) { List features = extractFeatures(output, form) if (features) { - List featureGeometries = features.collect { it.geometry } - Geometry geometry = GeometryUtils.getFeatureCollectionConvexHull(featureGeometries) String startDateInString = getStartDate(output) startDateInString = DateUtil.convertUTCDateToStringInTimeZone(startDateInString, clientTimeZone?:TimeZone.default) String name = "${form.name} site - ${startDateInString}" - geoJson = [ - type: 'Feature', - geometry: GeometryUtils.geometryToGeoJsonMap(geometry), - properties: [ - name: name, - description: "${name} (convex hull of all features)", - ], - features: features - ] + geoJson = createConvexHullGeoJSON(features, name) } } @@ -352,38 +310,60 @@ class ParatooProtocolConfig { private Map extractSiteDataFromPlotVisit(Map survey) { Map surveyData = getSurveyData(survey) - def plotLayoutId = getProperty(surveyData, plotLayoutIdPath) // Currently an int, may become uuid? + String plotLayoutId = getProperty(surveyData, plotLayoutIdPath) // Currently an int, may become uuid? if (!plotLayoutId) { log.warn("No plot_layout found in survey at path ${plotLayoutIdPath}") return null } + List plotLayoutPoints = getProperty(surveyData, plotLayoutPointsPath) Map plotSelection = getProperty(surveyData, plotSelectionPath) Map plotSelectionGeoJson = plotSelectionToGeoJson(plotSelection) - String plotLayoutDimensionLabel = getProperty(surveyData, plotLayoutDimensionLabelPath) String plotLayoutTypeLabel = getProperty(surveyData, plotLayoutTypeLabelPath) + List faunaPlotPoints = getProperty(surveyData, faunaPlotPointPath) - String name = plotSelectionGeoJson.properties.name + ' - ' + plotLayoutTypeLabel + ' (' + plotLayoutDimensionLabel + ')' + String name = plotSelectionGeoJson.properties.name + ' - ' + plotLayoutTypeLabel + ' ('+CORE_PLOT_SHORT+')' - Map plotGeoJson = createFeatureFromGeoJSON(plotLayoutPoints, name, plotLayoutId, plotSelectionGeoJson?.properties?.notes) + Map plotGeoJson = createFeatureFromGeoJSON(plotLayoutPoints, name, plotLayoutId, "${CORE_PLOT} ${plotSelectionGeoJson?.properties?.notes?:""}", CORE_PLOT_SHORT) - //Map faunaPlotGeoJson = toGeometry(plotLayout.fauna_plot_point) + if (faunaPlotPoints) { + name = plotSelectionGeoJson.properties.name + ' - ' + plotLayoutTypeLabel + ' (' + FAUNA_PLOT_SHORT + ')' + Map faunaPlotGeoJson = createFeatureFromGeoJSON(faunaPlotPoints, name, plotLayoutId, "${FAUNA_PLOT} ${plotSelectionGeoJson?.properties?.notes?:""}", FAUNA_PLOT_SHORT) + List features = [plotGeoJson, faunaPlotGeoJson] - // TODO maybe turn this into a feature with properties to distinguish the fauna plot? - // Or a multi-polygon? + name = plotSelectionGeoJson.properties.name + ' - ' + plotLayoutTypeLabel + ' (' + CORE_PLOT_SHORT + ' + ' + FAUNA_PLOT_SHORT + ')' + plotGeoJson = createConvexHullGeoJSON(features, name, plotLayoutId, plotGeoJson.properties.notes) + } plotGeoJson } - static Map createFeatureFromGeoJSON(List plotLayoutPoints, String name, def plotLayoutId, String notes = "") { + static Map createConvexHullGeoJSON (List features, String name, String externalId = "", String notes = "", String description = "") { + features = features.findAll { it.geometry != null } + List featureGeometries = features.collect { it.geometry } + Geometry geometry = GeometryUtils.getFeatureCollectionConvexHull(featureGeometries) + [ + type: 'Feature', + geometry: GeometryUtils.geometryToGeoJsonMap(geometry), + properties: [ + name: name, + externalId: externalId, + notes: notes, + description: "${description?:name}", + ], + features: features + ] + } + + static Map createFeatureFromGeoJSON(List plotLayoutPoints, String name, String plotLayoutId, String notes = "", String activityType = null) { Map plotGeometry = toGeometry(plotLayoutPoints) - createFeatureObject(plotGeometry, name, plotLayoutId, notes) + createFeatureObject(plotGeometry, name, plotLayoutId, notes, activityType) } - static Map createFeatureObject(Map plotGeometry, String name, plotLayoutId, String notes = "") { - [ + static Map createFeatureObject(Map plotGeometry, String name, String plotLayoutId, String notes = "", String activityType = null) { + Map featureObject = [ type : 'Feature', geometry : plotGeometry, properties: [ @@ -393,6 +373,10 @@ class ParatooProtocolConfig { notes : notes ] ] + if (activityType) { + featureObject.properties.activityType = activityType + } + featureObject } static Map toGeometry(List points) { @@ -407,7 +391,7 @@ class ParatooProtocolConfig { plotGeometry } - static Map createLineStringFeatureFromGeoJSON (List plotLayoutPoints, String name, def plotLayoutId, String notes = "") { + static Map createLineStringFeatureFromGeoJSON (List plotLayoutPoints, String name, String plotLayoutId, String notes = "") { Map plotGeometry = toLineStringGeometry(plotLayoutPoints) createFeatureObject(plotGeometry, name, plotLayoutId, notes) } diff --git a/src/main/groovy/au/org/ala/ecodata/reporting/ProjectXlsExporter.groovy b/src/main/groovy/au/org/ala/ecodata/reporting/ProjectXlsExporter.groovy index ff2e8e1cc..7d236ee70 100644 --- a/src/main/groovy/au/org/ala/ecodata/reporting/ProjectXlsExporter.groovy +++ b/src/main/groovy/au/org/ala/ecodata/reporting/ProjectXlsExporter.groovy @@ -176,6 +176,7 @@ class ProjectXlsExporter extends ProjectExporter { super(exporter) this.projectService = projectService distinctElectorates = new ArrayList() + useSpeciesUrlGetter = true setupManagementUnits(managementUnitService) setupFundingAbn(organisationService) setupProgramData(programService) @@ -185,6 +186,7 @@ class ProjectXlsExporter extends ProjectExporter { super(exporter, tabsToExport, [:], TimeZone.default) this.projectService = projectService this.formSectionPerTab = formSectionPerTab + useSpeciesUrlGetter = true addDataDescriptionToDownload(downloadMetadata) distinctElectorates = new ArrayList(electorates?:[]) distinctElectorates.sort() diff --git a/src/main/groovy/au/org/ala/ecodata/reporting/PropertyAccessor.groovy b/src/main/groovy/au/org/ala/ecodata/reporting/PropertyAccessor.groovy index 178a34e03..17b0ca042 100644 --- a/src/main/groovy/au/org/ala/ecodata/reporting/PropertyAccessor.groovy +++ b/src/main/groovy/au/org/ala/ecodata/reporting/PropertyAccessor.groovy @@ -84,7 +84,7 @@ class PropertyAccessor { results.each { item -> tmpResults.addAll(unrollSingle(item)) } - if (tmpResults.size > 0) { + if (tmpResults.size() > 0) { results = tmpResults } } diff --git a/src/main/groovy/au/org/ala/ecodata/reporting/ReportGroups.groovy b/src/main/groovy/au/org/ala/ecodata/reporting/ReportGroups.groovy index bbeeb7be5..d8f3ba5fd 100644 --- a/src/main/groovy/au/org/ala/ecodata/reporting/ReportGroups.groovy +++ b/src/main/groovy/au/org/ala/ecodata/reporting/ReportGroups.groovy @@ -119,6 +119,7 @@ class ReportGroups { static class DateGroup extends SinglePropertyGroupingStrategy { + static final String MISSING_DATE_GROUP_NAME = "Date missing" static DateTimeFormatter parser = ISODateTimeFormat.dateTimeNoMillis().withZone(DateTimeZone.default) DateTimeFormatter dateFormatter List buckets @@ -157,6 +158,10 @@ class ReportGroups { def group(data) { def value = propertyAccessor.getPropertyValue(data) + if (!value) { + return MISSING_DATE_GROUP_NAME // Use a special group for null / empty dates. + } + int result = bucketIndex(value) // we put results with an exact date match into the group where the end date of the bucket matches diff --git a/src/main/groovy/au/org/ala/ecodata/reporting/TabbedExporter.groovy b/src/main/groovy/au/org/ala/ecodata/reporting/TabbedExporter.groovy index 4664a347e..3151e2663 100644 --- a/src/main/groovy/au/org/ala/ecodata/reporting/TabbedExporter.groovy +++ b/src/main/groovy/au/org/ala/ecodata/reporting/TabbedExporter.groovy @@ -6,6 +6,7 @@ import au.org.ala.ecodata.metadata.OutputDateGetter import au.org.ala.ecodata.metadata.OutputMetadata import au.org.ala.ecodata.metadata.OutputModelProcessor import au.org.ala.ecodata.metadata.OutputNumberGetter +import au.org.ala.ecodata.metadata.SpeciesUrlGetter import grails.util.Holders import org.apache.commons.logging.Log import org.apache.commons.logging.LogFactory @@ -26,6 +27,7 @@ class TabbedExporter { ReportingService reportingService = Holders.grailsApplication.mainContext.getBean("reportingService") ActivityFormService activityFormService = Holders.grailsApplication.mainContext.getBean("activityFormService") OutputModelProcessor processor = new OutputModelProcessor() + String biePrefix = Holders.grailsApplication.config.getProperty("bie.url")+'species/' static String DATE_CELL_FORMAT = "dd/MM/yyyy" Map sheets @@ -35,6 +37,7 @@ class TabbedExporter { TimeZone timeZone Boolean useDateGetter = false Boolean useNumberGetter = false + boolean useSpeciesUrlGetter = false // These fields map full activity names to shortened names that are compatible with Excel tabs. protected Map activitySheetNames = [:] protected Map> typedActivitySheets = [:] @@ -232,6 +235,22 @@ class TabbedExporter { getter:new OutputNumberGetter(propertyPath, dataNode, documentMap, timeZone)] fieldConfiguration << field } + else if ((dataNode.dataType == 'species') && useSpeciesUrlGetter) { + // Return a property for the species name and a property for the species URL + Map nameField = field + [ + header:outputMetadata.getLabel(viewNode, dataNode), + property:propertyPath, + getter:new OutputDataGetter(propertyPath, dataNode, documentMap, timeZone)] + fieldConfiguration << nameField + + Map urlField = field + [ + description: "Link to species in the ALA", + header:outputMetadata.getLabel(viewNode, dataNode), + property:propertyPath, + getter:new SpeciesUrlGetter(propertyPath, dataNode, documentMap, timeZone, biePrefix) + ] + fieldConfiguration << urlField + } else { field += [ header:outputMetadata.getLabel(viewNode, dataNode), diff --git a/src/test/groovy/au/org/ala/ecodata/DataSetSummaryControllerSpec.groovy b/src/test/groovy/au/org/ala/ecodata/DataSetSummaryControllerSpec.groovy new file mode 100644 index 000000000..d7dc36f56 --- /dev/null +++ b/src/test/groovy/au/org/ala/ecodata/DataSetSummaryControllerSpec.groovy @@ -0,0 +1,90 @@ +package au.org.ala.ecodata + +import grails.testing.web.controllers.ControllerUnitTest +import org.apache.http.HttpStatus +import spock.lang.Specification + +class DataSetSummaryControllerSpec extends Specification implements ControllerUnitTest { + + ProjectService projectService = Mock(ProjectService) + def setup() { + controller.projectService = projectService + } + + def cleanup() { + } + + void "The update method delegates to the projectService"() { + setup: + String projectId = 'p1' + Map dataSetSummary = [dataSetId:'d1', name:'Data set 1'] + + when: + request.method = 'POST' + request.json = dataSetSummary + controller.update(projectId) + + then: + 1 * projectService.updateDataSet(projectId, dataSetSummary) >> [status:'ok'] + response.json == ['status':'ok'] + + } + + void "A project id must be specified either in the path or as part of the data set summary"() { + setup: + Map dataSetSummary = [dataSetId: 'd1', name: 'Data set 1'] + + when: + request.method = 'POST' + request.json = dataSetSummary + controller.update() + + then: + 0 * projectService.updateDataSet(_, _) + response.status == HttpStatus.SC_BAD_REQUEST + } + + void "The delete method delegates to the projectService"() { + setup: + String projectId = 'p1' + String dataSetSummaryId = 'd1' + + when: + request.method = 'DELETE' + controller.delete(projectId, dataSetSummaryId) + + then: + 1 * projectService.deleteDataSet(projectId, dataSetSummaryId) >> [status:'ok'] + response.json == ['status':'ok'] + } + + void "The bulkUpdate method delegates to the projectService"() { + setup: + String projectId = 'p1' + Map postBody = [dataSets:[[dataSetId:'d1', name:'Data set 1']]] + + when: + request.method = 'POST' + request.json = postBody + controller.bulkUpdate(projectId) + + then: + 1 * projectService.updateDataSets(projectId, postBody.dataSets) >> [status:'ok'] + response.json == ['status':'ok'] + } + + void "If a projectId is present in a dataSet it much match the projectId parameter in bulkUpdate"() { + setup: + String projectId = 'p1' + Map postBody = [dataSets:[[dataSetId:'d1', name:'Data set 1', projectId:'p1'], [dataSetId:'d2', name:'Data set 2', projectId:'p2']]] + + when: + request.method = 'POST' + request.json = postBody + controller.bulkUpdate(projectId) + + then: + 0 * projectService.updateDataSets(_, _) + response.status == HttpStatus.SC_BAD_REQUEST + } +} diff --git a/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy b/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy index f79449bdf..d1add60c5 100644 --- a/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy +++ b/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy @@ -26,6 +26,7 @@ class ParatooServiceSpec extends MongoSpec implements ServiceUnitTest> DUMMY_POLYGON @@ -138,19 +146,17 @@ class ParatooServiceSpec extends MongoSpec implements ServiceUnitTest> project - 1 * projectService.update(_, projectId, false) >> { data, pId, updateCollectory -> - Map dataSet = data.custom.dataSets[1] // The stubbed project already has a dataSet, so the new one will be index=1 + 1 * projectService.updateDataSet(projectId, _) >> { pId, dataSet -> + pId == projectId assert dataSet.surveyId != null assert dataSet.surveyId.eventTime != null assert dataSet.surveyId.userId == 'org1' @@ -158,7 +164,7 @@ class ParatooServiceSpec extends MongoSpec implements ServiceUnitTest> [resp: [collections: ["coarse-woody-debris-survey": [uuid: "1", createdAt: "2023-09-01T00:00:00.123Z", start_date_time: "2023-09-01T00:00:00.123Z", end_date_time: "2023-09-01T00:00:00.123Z"]]]] + 1 * webService.doPost(*_) >> [resp: [collections: ["coarse-woody-debris-survey": [uuid: "1", createdAt: "2023-09-01T00:00:00.123Z", start_date_time: "2023-09-01T00:00:00.123Z", end_date_time: "2023-09-01T00:00:00.123Z"], "coarse-woody-debris-survey-observation": [[point: [lat: 1, lng: 2, name: [data: [attributes: [symbol: "ab"]]]]]]]]] 1 * tokenService.getAuthToken(true) >> Mock(AccessToken) - 2 * projectService.get(projectId) >> [projectId: projectId, custom: [dataSets: [dataSet]]] - 1 * projectService.update([custom: [dataSets: [expectedDataSetAsync]]], 'p1', false) >> [status: 'ok'] - 1 * projectService.update([custom: [dataSets: [expectedDataSetSync]]], 'p1', false) >> [status: 'ok'] - 1 * activityService.create(_) >> [activityId: '123'] + 1 * projectService.updateDataSet(projectId, expectedDataSetAsync) >> [status: 'ok'] + 1 * projectService.updateDataSet(projectId, expectedDataSetSync) >> [status: 'ok'] + 1 * activityService.create({ + it.startDate == "2023-09-01T00:00:00Z" && it.endDate == "2023-09-01T00:00:00Z" && + it.plannedStartDate == "2023-09-01T00:00:00Z" && it.plannedEndDate == "2023-09-01T00:00:00Z" && + it.externalIds[0].externalId == "d1" && it.externalIds[0].idType == ExternalId.IdType.MONITOR_MINTED_COLLECTION_ID + }) >> [activityId: '123'] + 1 * siteService.create(_) >> { site = it[0]; [siteId: 's1'] } 1 * activityService.delete("123", true) >> [status: 'ok'] 1 * recordService.getAllByActivity('123') >> [] 1 * settingService.getSetting('paratoo.surveyData.mapping') >> { @@ -206,6 +217,7 @@ class ParatooServiceSpec extends MongoSpec implements ServiceUnitTest> [userId: userId] + 1 * userService.setCurrentUser(userId) and: result.updateResult == [status: 'ok'] + site.externalIds == [new ExternalId(externalId: "d1", idType: ExternalId.IdType.MONITOR_PLOT_GUID)] } void "The service can create a site from a submitted plot-selection"() { @@ -226,7 +240,7 @@ class ParatooServiceSpec extends MongoSpec implements ServiceUnitTest> [resp: surveyData] 1 * tokenService.getAuthToken(true) >> Mock(AccessToken) - 2 * projectService.update(_, projectId, false) >> [status: 'ok'] - 2 * projectService.get(projectId) >> [projectId: projectId, custom: [dataSets: [dataSet]]] + 2 * projectService.updateDataSet(projectId, _) >> [status: 'ok'] 1 * siteService.create(_) >> { site = it[0]; [siteId: 's1'] } - 1 * activityService.create(_) >> [activityId: '123'] + 1 * activityService.create({ + it.startDate == "2023-09-22T00:59:47Z" && it.endDate == "2023-09-23T00:59:47Z" && + it.plannedStartDate == "2023-09-22T00:59:47Z" && it.plannedEndDate == "2023-09-23T00:59:47Z" && + it.externalIds[0].externalId == "d1" && it.externalIds[0].idType == ExternalId.IdType.MONITOR_MINTED_COLLECTION_ID + }) >> [activityId: '123'] 1 * recordService.getAllByActivity('123') >> [] 1 * settingService.getSetting('paratoo.surveyData.mapping') >> { (["guid-3": [ @@ -322,12 +340,13 @@ class ParatooServiceSpec extends MongoSpec implements ServiceUnitTest> [userId: userId] + 1 * userService.setCurrentUser(userId) and: - site.name == "SATFLB0001 - Control (100 x 100)" - site.description == "SATFLB0001 - Control (100 x 100)" - site.notes == "some comment" - site.type == "surveyArea" + site.name == "SATFLB0001 - Control (Core + Fauna)" + site.description == "SATFLB0001 - Control (Core + Fauna)" + site.notes == "Core monitoring plot some comment" + site.type == "compound" site.publicationStatus == "published" site.externalIds[0].externalId == "2" site.externalIds[0].idType == ExternalId.IdType.MONITOR_PLOT_GUID @@ -336,6 +355,156 @@ class ParatooServiceSpec extends MongoSpec implements ServiceUnitTest", + "version_core": "" + ] + ) + ParatooCollectionId paratooCollectionId = buildCollectionId("mintCollectionIdBasalAreaPayload","guid-3") + Map dataSet = [dataSetId:'d1', grantId:'g1', surveyId:paratooCollectionId.toMap()] + ParatooProject project = new ParatooProject(id: projectId, project: new Project(projectId: projectId, custom: [dataSets: [dataSet]])) + Map surveyData = readSurveyData('basalAreaDbhReverseLookup') + + when: + Map result = service.submitCollection(collection, project) + waitAll(result.promise) + println ("finished waiting") + + then: + 1 * webService.doPost(*_) >> [resp: surveyData] + 1 * tokenService.getAuthToken(true) >> Mock(AccessToken) + 2 * projectService.updateDataSet(projectId, _) >> [status: 'ok'] + 0 * siteService.create(_) + 1 * activityService.create({ + it.startDate == "2023-09-22T00:59:47Z" && it.endDate == "2023-09-23T00:59:47Z" && + it.plannedStartDate == "2023-09-22T00:59:47Z" && it.plannedEndDate == "2023-09-23T00:59:47Z" && + it.externalIds[0].externalId == "d1" && it.externalIds[0].idType == ExternalId.IdType.MONITOR_MINTED_COLLECTION_ID + }) >> [activityId: '123'] + 1 * recordService.getAllByActivity('123') >> [] + 1 * settingService.getSetting('paratoo.surveyData.mapping') >> { + (["guid-3": [ + "name" : "Basal Area - DBH", + "usesPlotLayout": true, + "tags" : ["survey"], + "apiEndpoint" : "basal-area-dbh-measure-surveys", + "overrides" : [ + "dataModel": null, + "viewModel": null + ] + ]] as JSON).toString() + } + 1 * userService.getCurrentUserDetails() >> [userId: userId] + 1 * userService.setCurrentUser(userId) + } + + void "The service will create a new site if plot layout has been updated" () { + setup: + String projectId = 'p1' + String orgMintedId = 'd1' + Date afterSubmissionDate = DateUtil.parseWithMilliseconds("2023-09-15T06:00:11.996Z") + Date beforeSubmissionDate = DateUtil.parseWithMilliseconds("2023-09-14T06:00:11.996Z") + Site.withSession { session -> + new Site( + name: "SATFLB0001 - Control (Core)", + siteId: "s0", + extent: [geometry: DUMMY_POLYGON], + description: "SATFLB0001 - Control (Core)", + notes: "Core monitoring plot some comment", + type: "compound", + externalIds: [new ExternalId(externalId: "2", idType: ExternalId.IdType.MONITOR_PLOT_GUID)], + dateCreated: afterSubmissionDate, + lastUpdated: afterSubmissionDate + ).save(flush: true) + session.flush() + } + ParatooProtocolId protocol = new ParatooProtocolId(id: "1", version: 1) + ParatooCollection collection = new ParatooCollection( + orgMintedUUID:orgMintedId, + coreProvenance: [ + "system_core": "", + "version_core": "" + ] + ) + ParatooCollectionId paratooCollectionId = buildCollectionId("mintCollectionIdBasalAreaPayload","guid-3") + Map dataSet = [dataSetId:'d1', grantId:'g1', surveyId:paratooCollectionId.toMap()] + ParatooProject project = new ParatooProject(id: projectId, project: new Project(projectId: projectId, custom: [dataSets: [dataSet]])) + Map surveyData = readSurveyData('basalAreaDbhReverseLookup') + String date = DateUtil.format(new Date()) + date = date.replace("Z", ".999Z") + surveyData.collections."basal-area-dbh-measure-survey"."plot_visit"."plot_layout"."updatedAt" = date + Map site + + when: + def result = service.submitCollection(collection, project) + waitAll(result.promise) + + then: + 1 * webService.doPost(*_) >> [resp: surveyData] + 1 * tokenService.getAuthToken(true) >> Mock(AccessToken) + 2 * projectService.updateDataSet(projectId, _) >> [status: 'ok'] + 1 * siteService.create(_) >> { site = it[0]; [siteId: 's1'] } + 1 * activityService.create({ + it.startDate == "2023-09-22T00:59:47Z" && it.endDate == "2023-09-23T00:59:47Z" && + it.plannedStartDate == "2023-09-22T00:59:47Z" && it.plannedEndDate == "2023-09-23T00:59:47Z" && + it.externalIds[0].externalId == "d1" && it.externalIds[0].idType == ExternalId.IdType.MONITOR_MINTED_COLLECTION_ID + }) >> [activityId: '123'] + 1 * recordService.getAllByActivity('123') >> [] + 1 * settingService.getSetting('paratoo.surveyData.mapping') >> { + (["guid-3": [ + "name" : "Basal Area - DBH", + "usesPlotLayout": true, + "tags" : ["survey"], + "apiEndpoint" : "basal-area-dbh-measure-surveys", + "overrides" : [ + "dataModel": null, + "viewModel": null + ] + ]] as JSON).toString() + } + 1 * userService.getCurrentUserDetails() >> [userId: userId] + 1 * userService.setCurrentUser(userId) + result.updateResult == [status: 'ok'] + } + + void "isUpdatedPlotLayout should check plot layout has been updated after site has been updated" () { + given: + def date1 = DateUtil.parseWithMilliseconds("2023-09-22T00:59:47.111Z") + def date2 = DateUtil.parseWithMilliseconds("2023-09-23T00:59:47.111Z") + def date3 = DateUtil.parseWithMilliseconds("2023-09-24T00:59:47.111Z") + + when: + def result = service.isUpdatedPlotLayout(date1, date2) + + then: + result + + when: + result = service.isUpdatedPlotLayout(date3, date2) + + then: + !result + } + private Map getProject(){ [ projectId:"p1", @@ -364,7 +533,7 @@ class ParatooServiceSpec extends MongoSpec implements ServiceUnitTest> [projectArea, plot] @@ -374,6 +543,10 @@ class ParatooServiceSpec extends MongoSpec implements ServiceUnitTest> null - - when: // no scientific name - result = service.transformSpeciesName("Frogs [Class] (scientific: )") - outputSpeciesId = result.remove("outputSpeciesId") - - then: - outputSpeciesId != null - result == [name: "Frogs", scientificName: "Frogs", guid: "A_GUID", commonName: "Frogs", taxonRank: "Class"] - 2 * metadataService.autoPopulateSpeciesData(_) >> null - } - void "buildRelationshipTree should build relationship tree correctly"() { given: def properties = [ @@ -561,6 +722,38 @@ class ParatooServiceSpec extends MongoSpec implements ServiceUnitTest> null + 1 * speciesReMatchService.searchByName(_, false, true) >> null + + when: // no scientific name + result = service.transformSpeciesName("Frogs [Class] (scientific: )") + outputSpeciesId = result.remove("outputSpeciesId") + + then: + outputSpeciesId != null + result == [name: "Frogs", scientificName: "", guid: "A_GUID", commonName: "Frogs", taxonRank: "Class"] + 1 * speciesReMatchService.searchByName(_) >> null + 1 * speciesReMatchService.searchByName(_, false, true) >> null + + when: // Do not create record when value equals special cases. Therefore, removes guid. + result = service.transformSpeciesName("Other") + outputSpeciesId = result.remove("outputSpeciesId") + + then: + outputSpeciesId != null + result == [name: "Other", scientificName: null, commonName: "Other", taxonRank: null] + 1 * speciesReMatchService.searchByName(_) >> null + 1 * speciesReMatchService.searchByName(_, false, true) >> null + + } + void "buildTreeFromParentChildRelationships should build tree correctly"() { given: def relationships = [ @@ -1237,13 +1430,19 @@ class ParatooServiceSpec extends MongoSpec implements ServiceUnitTest> [ + commonName: "Cat", + scientificName: "Felis catus", + guid: "TAXON_ID", + taxonRank: "species" + ] result == [ lut: [ commonName: "Cat", - name: "Cat", - taxonRank: null, - scientificName: "Cat", - guid: "A_GUID" + name: "Felis catus (Cat)", + taxonRank: "species", + scientificName: "Felis catus", + guid: "TAXON_ID" ] ] @@ -1255,18 +1454,25 @@ class ParatooServiceSpec extends MongoSpec implements ServiceUnitTest> null + 1 * speciesReMatchService.searchByName("Cats", false, true) >> [ + commonName: "Cat", + scientificName: "Felis catus", + guid: "TAXON_ID", + taxonRank: "species" + ] result == [ lut: [ - commonName: "Cat", - name: "Cat", - taxonRank: null, - scientificName: "Cat", - guid: "A_GUID" + commonName: "Cats", + name: "Felis catus (Cats)", + taxonRank: "species", + scientificName: "Felis catus", + guid: "TAXON_ID" ] ] } @@ -1330,6 +1536,130 @@ class ParatooServiceSpec extends MongoSpec implements ServiceUnitTest { ProjectActivityService projectActivityServiceStub = Stub(ProjectActivityService) @@ -761,4 +765,139 @@ class ProjectServiceSpec extends MongoSpec implements ServiceUnitTest } + def "Sites can be listed by externalId and sorted"() { + when: + def result + Site.withSession { session -> + result = service.create([name:'Site 1', siteId:"s1", externalIds:[new ExternalId(externalId:'e1', idType:ExternalId.IdType.MONITOR_PLOT_GUID)]]) + session.flush() + result = service.create([name:'Site 2', siteId:"s2", externalIds:[new ExternalId(externalId:'e1', idType:ExternalId.IdType.MONITOR_PLOT_GUID)]]) + session.flush() + } + then: + def sites = Site.findAllByExternalId(ExternalId.IdType.MONITOR_PLOT_GUID, 'e1', ['sort': "lastUpdated", 'order': "desc"]) + sites.size() == 2 + sites[0].name == 'Site 2' + sites[0].externalIds.size() == 1 + sites.externalIds.externalId == [['e1'], ['e1']] + sites.externalIds.idType == [[ExternalId.IdType.MONITOR_PLOT_GUID], [ExternalId.IdType.MONITOR_PLOT_GUID]] + } + + def "The site area is calculated from the FeatureCollection for a compound site"() { + setup: + def coordinates = [[148.260498046875, -37.26530995561874], [148.260498046875, -37.26531995561874], [148.310693359375, -37.26531995561874], [148.310693359375, -37.26531995561874], [148.260498046875, -37.26530995561874]] + def extent = buildExtent('drawn', 'Polygon', coordinates) + Map site = [type: Site.TYPE_COMPOUND, extent: extent, features: [ + [ + type : "Feature", + geometry: [ + type : "Polygon", + coordinates: coordinates + ] + ], + [ + type : "Feature", + geometry: [ + type : "Polygon", + coordinates: coordinates + ] + ] + ]] + + when: + service.populateLocationMetadataForSite(site) + + then: + 1 * spatialServiceMock.intersectGeometry(_, _) >> [:] + site.extent.geometry.aream2 == 4938.9846950349165d + + when: + site.type = Site.TYPE_WORKS_AREA + service.populateLocationMetadataForSite(site) + + then: + 1 * spatialServiceMock.intersectGeometry(_, _) >> [:] + site.extent.geometry.aream2 == 2469.492347517461 + + } private Map buildExtent(source, type, coordinates, pid = '') { return [source:source, geometry:[type:type, coordinates: coordinates, pid:pid]] diff --git a/src/test/groovy/au/org/ala/ecodata/SpeciesReMatchServiceSpec.groovy b/src/test/groovy/au/org/ala/ecodata/SpeciesReMatchServiceSpec.groovy index c848689b5..159c70f22 100644 --- a/src/test/groovy/au/org/ala/ecodata/SpeciesReMatchServiceSpec.groovy +++ b/src/test/groovy/au/org/ala/ecodata/SpeciesReMatchServiceSpec.groovy @@ -67,4 +67,69 @@ class SpeciesReMatchServiceSpec extends Specification implements ServiceUnitTest result2 == resp2 } + void "search name server by name" () { + setup: + grailsApplication.config.namesmatching.url = "http://localhost:8080/" + grailsApplication.config.namesmatching.strategy = ["exactMatch", "vernacularMatch"] + def resp = [ + "success": true, + "scientificName": "Red", + "taxonConceptID": "ALA_DR22913_1168_0", + "rank": "genus", + "rankID": 6000, + "lft": 24693, + "rgt": 24693, + "matchType": "higherMatch", + "nameType": "SCIENTIFIC", + "kingdom": "Bamfordvirae", + "kingdomID": "https://www.catalogueoflife.org/data/taxon/8TRHY", + "phylum": "Nucleocytoviricota", + "phylumID": "https://www.catalogueoflife.org/data/taxon/5G", + "classs": "Megaviricetes", + "classID": "https://www.catalogueoflife.org/data/taxon/6224M", + "order": "Pimascovirales", + "orderID": "https://www.catalogueoflife.org/data/taxon/623FC", + "family": "Iridoviridae", + "familyID": "https://www.catalogueoflife.org/data/taxon/BFM", + "genus": "Red", + "genusID": "ALA_DR22913_1168_0", + "issues": [ + "noIssue" + ] + ] + service.webService.getJson({it.contains("search?q=")}) >> resp + when: + def result = service.searchByName("name") + + then: + result == null + + when: + resp.matchType = "exactMatch" + def result2 = service.searchByName("name") + + then: + service.webService.getJson({it.contains("search?q=")}) >> resp + result2 == [ + scientificName: "Red", + commonName: null, + guid: "ALA_DR22913_1168_0", + taxonRank: "genus" + ] + + when: + resp.matchType = "vernacularMatch" + def result3 = service.searchByName("name", false, true) + + then: + service.webService.getJson({it.contains("searchByVernacularName")}) >> resp + result3 == [ + scientificName: "Red", + commonName: null, + guid: "ALA_DR22913_1168_0", + taxonRank: "genus" + ] + + } + } diff --git a/src/test/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfigSpec.groovy b/src/test/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfigSpec.groovy index dbfbf87fa..62d2b2efb 100644 --- a/src/test/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfigSpec.groovy +++ b/src/test/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfigSpec.groovy @@ -100,9 +100,7 @@ class ParatooProtocolConfigSpec extends Specification { Map observation = apiOutput.collections Map floristicsSurveyConfig = [ apiEndpoint:'floristics-veg-survey-lites', - usesPlotLayout:true, - startDatePath: 'plot_visit.start_date', - endDatePath: 'plot_visit.end_date' + usesPlotLayout:true ] ParatooProtocolConfig config = new ParatooProtocolConfig(floristicsSurveyConfig) config.setSurveyId(ParatooCollectionId.fromMap([survey_metadata: apiOutput.survey_metadata])) @@ -134,9 +132,16 @@ class ParatooProtocolConfigSpec extends Specification { transformData(observation, activityForm, config) then: - config.getStartDate(observation) == "2024-04-08T01:23:28Z" - config.getEndDate(observation) == "2024-04-10T01:23:28Z" - config.getGeoJson(observation) == [type: "Feature", geometry: [type: "Polygon", coordinates: [[[152.880694, -27.388252], [152.880651, -27.388336], [152.880518, -27.388483], [152.880389, -27.388611], [152.88028, -27.388749], [152.880154, -27.388903], [152.880835, -27.389463], [152.880644, -27.389366], [152.880525, -27.389248], [152.88035, -27.389158], [152.880195, -27.389021], [152.880195, -27.389373], [152.880797, -27.388316], [152.881448, -27.388909], [152.881503, -27.388821], [152.881422, -27.388766], [152.881263, -27.388644], [152.881107, -27.388549], [152.880939, -27.388445], [152.881314, -27.389035], [152.88122, -27.389208], [152.881089, -27.389343], [152.880973, -27.389472], [152.880916, -27.389553], [152.880694, -27.388252]]]], properties: [name: "QDASEQ0001 - Control (100 x 100)", externalId: 1, description: "QDASEQ0001 - Control (100 x 100)", notes: "some comment"]] + config.getStartDate(observation) == "2022-09-21T01:55:44Z" + config.getEndDate(observation) == "2022-09-21T01:55:44Z" + config.getGeoJson(observation) == [ + type: "Feature", + geometry: [ + type: "Polygon", + coordinates: [[[152.880694, -27.388252], [152.880651, -27.388336], [152.880518, -27.388483], [152.880389, -27.388611], [152.88028, -27.388749], [152.880154, -27.388903], [152.880835, -27.389463], [152.880644, -27.389366], [152.880525, -27.389248], [152.88035, -27.389158], [152.880195, -27.389021], [152.880195, -27.389373], [152.880797, -27.388316], [152.881448, -27.388909], [152.881503, -27.388821], [152.881422, -27.388766], [152.881263, -27.388644], [152.881107, -27.388549], [152.880939, -27.388445], [152.881314, -27.389035], [152.88122, -27.389208], [152.881089, -27.389343], [152.880973, -27.389472], [152.880916, -27.389553], [152.880694, -27.388252]]] + ], + properties: [name: "QDASEQ0001 - Control (Core)", externalId: "1", description: "QDASEQ0001 - Control (Core)", notes: "Core monitoring plot some comment", activityType:'Core'] + ] } def "The basal-area-dbh-measure-survey can be used with this config"() { @@ -144,9 +149,7 @@ class ParatooProtocolConfigSpec extends Specification { Map surveyData = readSurveyData('basalAreaDbhReverseLookup') Map basalAreaDbhMeasureSurveyConfig = [ apiEndpoint:'basal-area-dbh-measure-surveys', - usesPlotLayout:true, - startDatePath: 'start_date', - endDatePath: 'start_date', + usesPlotLayout:true ] ParatooProtocolConfig config = new ParatooProtocolConfig(basalAreaDbhMeasureSurveyConfig) config.setSurveyId(ParatooCollectionId.fromMap([survey_metadata: surveyData.survey_metadata])) @@ -185,23 +188,65 @@ class ParatooProtocolConfigSpec extends Specification { transformData(observation, activityForm, config) expect: - config.getStartDate(observation) == "2024-03-28T03:17:01Z" - config.getEndDate(observation) == "2024-03-28T03:17:01Z" + config.getStartDate(observation) == "2023-09-22T00:59:47Z" + config.getEndDate(observation) == "2023-09-23T00:59:47Z" config.getGeoJson(observation) == [ - type : "Feature", - geometry : [ - type : "Polygon", - coordinates: [[[138.63720760798054, -34.97222197296049], [138.63720760798054, -34.97204230990367], [138.63720760798054, -34.971862646846844], [138.63720760798054, -34.97168298379002], [138.63720760798054, -34.9715033207332], [138.63720760798054, -34.971413489204785], [138.63731723494544, -34.971413489204785], [138.6375364888752, -34.971413489204785], [138.63775574280498, -34.971413489204785], [138.63797499673475, -34.971413489204785], [138.63819425066453, -34.971413489204785], [138.63830387762943, -34.971413489204785], [138.63830387762943, -34.9715033207332], [138.63830387762943, -34.97168298379002], [138.63830387762943, -34.971862646846844], [138.63830387762943, -34.97204230990367], [138.63830387762943, -34.97222197296049], [138.63830387762943, -34.9723118044889], [138.63819425066453, -34.9723118044889], [138.63797499673475, -34.9723118044889], [138.63775574280498, -34.9723118044889], [138.6375364888752, -34.9723118044889], [138.63731723494544, -34.9723118044889], [138.63720760798054, -34.9723118044889], [138.63720760798054, -34.97222197296049]]]], - properties: ["name": "SATFLB0001 - Control (100 x 100)", externalId: 2, description: "SATFLB0001 - Control (100 x 100)", notes: "some comment"] + type:"Feature", + geometry:[ + coordinates:[[[138.6372, -34.9723], [138.6371, -34.9723], [138.6371, -34.9714], [138.6382, -34.9714], [138.6383, -34.9714], [138.6383, -34.9723], [138.6372, -34.9723]]], + type:"Polygon" + ], + properties:[name:"SATFLB0001 - Control (Core + Fauna)", externalId:"2", notes:"Core monitoring plot some comment", description:"SATFLB0001 - Control (Core + Fauna)"], + features:[ + [ + type:"Feature", + geometry:[ + type:"Polygon", + coordinates:[[[138.63720760798054, -34.97222197296049], [138.63720760798054, -34.97204230990367], [138.63720760798054, -34.971862646846844], [138.63720760798054, -34.97168298379002], [138.63720760798054, -34.9715033207332], [138.63720760798054, -34.971413489204785], [138.63731723494544, -34.971413489204785], [138.6375364888752, -34.971413489204785], [138.63775574280498, -34.971413489204785], [138.63797499673475, -34.971413489204785], [138.63819425066453, -34.971413489204785], [138.63830387762943, -34.971413489204785], [138.63830387762943, -34.9715033207332], [138.63830387762943, -34.97168298379002], [138.63830387762943, -34.971862646846844], [138.63830387762943, -34.97204230990367], [138.63830387762943, -34.97222197296049], [138.63830387762943, -34.9723118044889], [138.63819425066453, -34.9723118044889], [138.63797499673475, -34.9723118044889], [138.63775574280498, -34.9723118044889], [138.6375364888752, -34.9723118044889], [138.63731723494544, -34.9723118044889], [138.63720760798054, -34.9723118044889], [138.63720760798054, -34.97222197296049]]] + ], + properties:[name:"SATFLB0001 - Control (Core)", externalId:"2", description:"SATFLB0001 - Control (Core)", notes:"Core monitoring plot some comment", activityType:'Core']], + [ + type:"Feature", + geometry:[ + type:"Polygon", + coordinates:[[[138.6371026907952, -34.971403261821905], [138.63709732396242, -34.972304399720215], [138.6381916652405, -34.972304399720215], [138.63819166764344, -34.9714076576406], [138.6371026907952, -34.971403261821905]]] + ], + properties:[ + name:"SATFLB0001 - Control (Fauna)", externalId:"2", description:"SATFLB0001 - Control (Fauna)", notes:"Fauna plot some comment", activityType:'Fauna' + ] + ] + ] ] - config.getGeoJson(observation, activityForm).features == [[ - type : "Feature", - geometry : [ - type : "Point", - coordinates: [149.0651491, -35.2592444] + config.getGeoJson(observation, activityForm).features == [ + [ + type:"Feature", + geometry:[ + type:"Polygon", + coordinates:[[[138.63720760798054, -34.97222197296049], [138.63720760798054, -34.97204230990367], [138.63720760798054, -34.971862646846844], [138.63720760798054, -34.97168298379002], [138.63720760798054, -34.9715033207332], [138.63720760798054, -34.971413489204785], [138.63731723494544, -34.971413489204785], [138.6375364888752, -34.971413489204785], [138.63775574280498, -34.971413489204785], [138.63797499673475, -34.971413489204785], [138.63819425066453, -34.971413489204785], [138.63830387762943, -34.971413489204785], [138.63830387762943, -34.9715033207332], [138.63830387762943, -34.97168298379002], [138.63830387762943, -34.971862646846844], [138.63830387762943, -34.97204230990367], [138.63830387762943, -34.97222197296049], [138.63830387762943, -34.9723118044889], [138.63819425066453, -34.9723118044889], [138.63797499673475, -34.9723118044889], [138.63775574280498, -34.9723118044889], [138.6375364888752, -34.9723118044889], [138.63731723494544, -34.9723118044889], [138.63720760798054, -34.9723118044889], [138.63720760798054, -34.97222197296049]]] + ], + properties:[ + name:"SATFLB0001 - Control (Core)", + externalId:"2", + description:"SATFLB0001 - Control (Core)", + notes:"Core monitoring plot some comment", + activityType: 'Core' + ] ], - properties: ["name": "Point aParatooForm 2-1", externalId: 37, id: "aParatooForm 2-1"] - ]] + [ + type:"Feature", + geometry:[ + type:"Polygon", + coordinates:[[[138.6371026907952, -34.971403261821905], [138.63709732396242, -34.972304399720215], [138.6381916652405, -34.972304399720215], [138.63819166764344, -34.9714076576406], [138.6371026907952, -34.971403261821905]]] + ], + properties:[ + name:"SATFLB0001 - Control (Fauna)", + externalId:"2", + description:"SATFLB0001 - Control (Fauna)", + notes:"Fauna plot some comment", + activityType: 'Fauna' + ] + ] + ] } def "The observations from opportunistic-survey can be filtered"() { @@ -211,9 +256,7 @@ class ParatooProtocolConfigSpec extends Specification { Map opportunisticSurveyConfig = [ apiEndpoint : 'opportunistic-surveys', usesPlotLayout: false, - geometryType : 'Point', - startDatePath : 'start_date_time', - endDatePath : 'end_date_time' + geometryType : 'Point' ] ActivityForm activityForm = new ActivityForm( name: "aParatooForm 1", @@ -271,7 +314,9 @@ class ParatooProtocolConfigSpec extends Specification { features : [[type: "Feature", geometry: [type: "Point", coordinates: [138.63, -35.0005]], properties:[name:"Point aParatooForm 1-1", externalId:40, id:"aParatooForm 1-1"]]], properties: [ name : "aParatooForm 1 site - ${startDateInDefaultTimeZone}", - description: "aParatooForm 1 site - ${startDateInDefaultTimeZone} (convex hull of all features)" + description: "aParatooForm 1 site - ${startDateInDefaultTimeZone}", + externalId: "", + notes: "", ] ] } @@ -287,7 +332,7 @@ class ParatooProtocolConfigSpec extends Specification { ] when: - result = ParatooProtocolConfig.createLineStringFeatureFromGeoJSON([[lat: 1, lng: 2], [lat: 3, lng: 4], [lat: 5, lng: 6]], "test name", 1, "test notes") + result = ParatooProtocolConfig.createLineStringFeatureFromGeoJSON([[lat: 1, lng: 2], [lat: 3, lng: 4], [lat: 5, lng: 6]], "test name", "1", "test notes") then: result == [ @@ -298,13 +343,90 @@ class ParatooProtocolConfigSpec extends Specification { ], "properties": [ "name" : "test name", - "externalId" : 1, + "externalId" : "1", "description": "test name", "notes" : "test notes" ] ] } + def "getPlotLayoutUpdatedAt should get plot layout's updatedAt property" () { + given: + def surveyData = readSurveyData('floristicsStandardReverseLookup') + Map floristicsSurveyConfig = [ + apiEndpoint:'floristics-veg-survey-lites', + usesPlotLayout:true + ] + ParatooProtocolConfig config = new ParatooProtocolConfig(floristicsSurveyConfig) + config.setSurveyId(ParatooCollectionId.fromMap([survey_metadata: surveyData.survey_metadata])) + + when: + def result = config.getPlotLayoutUpdatedAt(surveyData.collections) + + then: + result.equals(DateUtil.parseWithMilliseconds("2023-09-14T06:00:11.473Z")) + } + + def "createConvexHullGeoJSON should create convex hull based on valid geometries" () { + + given: + def features = [ + [ + type: "Feature", + geometry: [ + type: "Polygon", + coordinates: [[[1, 1], [1, 2], [2, 2], [2, 1], [1, 1]]] + ] + ], + [ + type: "Feature", + geometry: [ + type: "Polygon", + coordinates: [[[1, 1], [1, 2], [2, 2], [2, 1], [1, 1]]] + ] + ], + [ + type: "Feature", + geometry: null + ] + ] + + when: + def result = ParatooProtocolConfig.createConvexHullGeoJSON(features, "test name", "1", "test notes") + + then: + result == [ + type: "Feature", + geometry: [ + type: "Polygon", + coordinates: [[[1, 1], [1, 2], [2, 2], [2, 1], [1, 1]]] + ], + properties: [ + name: "test name", + externalId: "1", + description: "test name", + notes: "test notes" + ], + features:[ + [ + type: "Feature", + geometry: [ + type: "Polygon", + coordinates: [[[1, 1], [1, 2], [2, 2], [2, 1], [1, 1]]] + ] + ], + [ + type: "Feature", + geometry: [ + type: "Polygon", + coordinates: [[[1, 1], [1, 2], [2, 2], [2, 1], [1, 1]]] + ] + ] + ] + ] + + } + def transformData(Map surveyDataAndObservations, ActivityForm form, ParatooProtocolConfig config) { ParatooService.addPlotDataToObservations(surveyDataAndObservations, config) paratooService.rearrangeSurveyData(surveyDataAndObservations, surveyDataAndObservations, form.sections[0].template.relationships.ecodata, form.sections[0].template.relationships.apiOutput) diff --git a/src/test/groovy/au/org/ala/ecodata/reporting/ReportGroupsSpec.groovy b/src/test/groovy/au/org/ala/ecodata/reporting/ReportGroupsSpec.groovy index 81679f6fd..11408dbda 100644 --- a/src/test/groovy/au/org/ala/ecodata/reporting/ReportGroupsSpec.groovy +++ b/src/test/groovy/au/org/ala/ecodata/reporting/ReportGroupsSpec.groovy @@ -105,6 +105,8 @@ public class ReportGroupsSpec extends Specification { ['2014-09-01T00:00:00Z', '2014-10-01T00:00:00Z', '2014-11-01T00:00:00Z'] | '2014-08-13T00:00:00Z' | 'MM-yyyy' | 'Before 09-2014' ['2014-09-01T00:00:00Z', '2014-10-01T00:00:00Z', '2014-11-01T00:00:00Z'] | '2014-11-01T00:00:01Z' | 'MM-yyyy' | 'After 10-2014' ['2014-10-01T00:00:00Z', '2015-01-01T00:00:00Z', '2015-04-01T00:00:00Z'] | '2014-10-13T00:00:00Z' | 'MM-yyyy' | '10-2014 - 12-2014' + ['2014-10-01T00:00:00Z', '2015-01-01T00:00:00Z', '2015-04-01T00:00:00Z'] | null | 'MM-yyyy' | 'Date missing' + ['2014-10-01T00:00:00Z', '2015-01-01T00:00:00Z', '2015-04-01T00:00:00Z'] | '' | 'MM-yyyy' | 'Date missing' } diff --git a/src/test/groovy/au/org/ala/ecodata/reporting/TabbedExporterSpec.groovy b/src/test/groovy/au/org/ala/ecodata/reporting/TabbedExporterSpec.groovy index 81d261aa5..695749d37 100644 --- a/src/test/groovy/au/org/ala/ecodata/reporting/TabbedExporterSpec.groovy +++ b/src/test/groovy/au/org/ala/ecodata/reporting/TabbedExporterSpec.groovy @@ -1,6 +1,8 @@ package au.org.ala.ecodata.reporting import au.org.ala.ecodata.* +import au.org.ala.ecodata.metadata.OutputDataGetter +import au.org.ala.ecodata.metadata.SpeciesUrlGetter import grails.testing.gorm.DomainUnitTest import grails.testing.web.GrailsWebUnitTest import grails.util.Holders @@ -97,6 +99,49 @@ class TabbedExporterSpec extends Specification implements GrailsWebUnitTest, Dom } + def "Species data types will be expanded into two export columns if useSpeciesUrlGetter is true"() { + setup: + String type = 'form' + ActivityForm form = buildMockForm(type, buildFormTemplateWithSpecies()) + + when: + tabbedExporter.useSpeciesUrlGetter = true + List config = tabbedExporter.getActivityExportConfig(type, true) + + then: + 1 * activityFormService.findVersionedActivityForm(type) >> [form] + + config.size() == 3 + config[1].header == 'Species label' + config[1].property == 'form.species' + config[1].getter instanceof OutputDataGetter + + config[2].header == 'Species label' + config[2].property == 'form.species' + config[2].getter instanceof SpeciesUrlGetter + + } + + def "Species data types will only export the species name if useSpeciesUrlGetter is false"() { + setup: + String type = 'form' + ActivityForm form = buildMockForm(type, buildFormTemplateWithSpecies()) + + when: + tabbedExporter.useSpeciesUrlGetter = false + List config = tabbedExporter.getActivityExportConfig(type, true) + + then: + 1 * activityFormService.findVersionedActivityForm(type) >> [form] + + config.size() == 2 + config[1].header == 'Species label' + config[1].property == 'form.species' + config[1].getter instanceof OutputDataGetter + } + + + private ActivityForm buildMockForm(String name, Map template) { ActivityForm form = new ActivityForm(name:name, formVersion:1) FormSection section = new FormSection(name:name, template:template) @@ -142,4 +187,25 @@ class TabbedExporterSpec extends Specification implements GrailsWebUnitTest, Dom ] ] } + + private Map buildFormTemplateWithSpecies() { + [ + dataModel:[ + [ + name:"species", + dataType:"species" + ] + ], + viewModel:[ + [type:'row', items:[ + [ + type:'speciesSelect', + source:'species', + preLabel:'Species label' + ] + ]] + + ] + ] + } } diff --git a/src/test/resources/paratoo/basalAreaDbhReverseLookup.json b/src/test/resources/paratoo/basalAreaDbhReverseLookup.json index 27145c2ab..15ab3ed80 100644 --- a/src/test/resources/paratoo/basalAreaDbhReverseLookup.json +++ b/src/test/resources/paratoo/basalAreaDbhReverseLookup.json @@ -41,7 +41,8 @@ "system_org": "MERIT" } }, - "start_date": "2023-09-22T00:59:47.807Z", + "start_date_time": "2023-09-22T00:59:47.807Z", + "end_date_time": "2023-09-23T00:59:47.807Z", "createdAt": "2023-09-22T01:05:19.981Z", "updatedAt": "2023-09-22T01:05:19.981Z", "plot_size": {