Skip to content

Commit

Permalink
#941
Browse files Browse the repository at this point in the history 
- fixes NPE when calculation convex hull
- new site for edited site
  • Loading branch information
@temi
temi committed May 16, 2024
1 parent e815077 commit 93ad03d
Show file tree
Hide file tree
Showing 7 changed files with 264 additions and 36 deletions.
2 changes: 1 addition & 1 deletion grails-app/controllers/au/org/ala/ecodata/AdminController.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -737,7 +737,7 @@ class AdminController {
def reSubmitDataSet() {
String projectId = params.id
String dataSetId = params.dataSetId
String userId = params.userId ?: userService.getCurrentUser().userId
String userId = params.userId ?: userService.currentUser()?.userId
if (!projectId || !dataSetId || !userId) {
render text: [message: "Bad request"] as JSON, status: HttpStatus.SC_BAD_REQUEST
return
Expand Down
39 changes: 30 additions & 9 deletions grails-app/services/au/org/ala/ecodata/ParatooService.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -279,9 +279,14 @@ class ParatooService {

String siteName = null
if (!dataSet.siteId) {
Map site = createSiteFromSurveyData(surveyDataAndObservations, collection, surveyId, project.project, config, form)
dataSet.siteId = site.siteId
siteName = site.name
try {
Map site = createSiteFromSurveyData(surveyDataAndObservations, collection, surveyId, project.project, config, form)
dataSet.siteId = site.siteId
siteName = site.name
}
catch (Exception ex) {
log.error("Error creating site for ${collection.orgMintedUUID}: ${ex.message}")
}
}

// plot layout is of type geoMap. Therefore, expects a site id.
Expand Down Expand Up @@ -584,6 +589,7 @@ class ParatooService {

private Map createSiteFromSurveyData(Map observation, ParatooCollection collection, ParatooCollectionId surveyId, Project project, ParatooProtocolConfig config, ActivityForm form) {
String siteId = null
Date updatedPlotLayoutDate
// Create a site representing the location of the collection
Map siteProps = null
Map geoJson = config.getGeoJson(observation, form)
Expand All @@ -605,16 +611,22 @@ class ParatooService {
Site site
// create new site for every non-plot submission
if (config.usesPlotLayout) {
updatedPlotLayoutDate = config.getPlotLayoutUpdatedAt(observation)
List sites = Site.findAllByExternalId(ExternalId.IdType.MONITOR_PLOT_GUID, externalId, [sort: "lastUpdated", order: "desc"])
if (sites)
site = sites.first()
}

Map result
// If the plot layout has been updated, create a new site
if (!site || isUpdatedPlotLayout(site, observation, config)) {
if (!site) {
result = siteService.create(siteProps)
} else {
}
else if(isUpdatedPlotLayout(site.lastUpdated, updatedPlotLayoutDate)){
siteProps.name = "${siteProps.name} - ${DateUtil.formatAsDisplayDateTime(updatedPlotLayoutDate)}"
result = siteService.create(siteProps)
}
else {
result = [siteId: site.siteId]
}

Expand All @@ -627,10 +639,19 @@ class ParatooService {
[siteId:siteId, name:siteProps?.name]
}

private static boolean isUpdatedPlotLayout (Site site, Map observation, ParatooProtocolConfig config) {
Date localSiteUpdated = site.lastUpdated
Date plotLayoutUpdated = config.getPlotLayoutUpdatedAt(observation)
plotLayoutUpdated?.after(localSiteUpdated) ?: false
/**
* check if the plot layout has been updated after site has been updated. This means user has edited plot layout and
* a new site should be created.
* @param siteLastUpdated
* @param plotLayoutLastUpdated
* @return
*/
static boolean isUpdatedPlotLayout (Date siteLastUpdated, Date plotLayoutLastUpdated) {
if ((siteLastUpdated != null) && (plotLayoutLastUpdated != null)) {
return plotLayoutLastUpdated.after(siteLastUpdated)
}

return false
}

private Map syncParatooProtocols(List<Map> protocols) {
Expand Down
30 changes: 15 additions & 15 deletions src/main/groovy/au/org/ala/ecodata/converter/FeatureConverter.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,27 +4,27 @@ class FeatureConverter implements RecordFieldConverter {

List<Map> convert(Map data, Map metadata = [:]) {
Map record = [:]
if (data[metadata.name]) {
Double latitude = getDecimalLatitude(data[metadata.name])
Double longitude = getDecimalLongitude(data[metadata.name])

// Don't override decimalLongitud or decimalLatitude in case they are null, site info could've already set them
if (latitude != null) {
record.decimalLatitude = latitude
}

Double latitude = getDecimalLatitude(data[metadata.name])
Double longitude = getDecimalLongitude(data[metadata.name])

// Don't override decimalLongitud or decimalLatitude in case they are null, site info could've already set them
if(latitude != null) {
record.decimalLatitude = latitude
}

if (longitude != null) {
record.decimalLongitude = longitude
}
if (longitude != null) {
record.decimalLongitude = longitude
}


Map dwcMappings = extractDwcMapping(metadata)
Map dwcMappings = extractDwcMapping(metadata)

record << getDwcAttributes(data, dwcMappings)
record << getDwcAttributes(data, dwcMappings)

if (data.dwcAttribute) {
record[data.dwcAttribute] = data.value
if (data.dwcAttribute) {
record[data.dwcAttribute] = data.value
}
}

[record]
Expand Down
4 changes: 2 additions & 2 deletions src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -335,7 +335,7 @@ class ParatooProtocolConfig {
plotGeoJson
}

static Map createConvexHullGeoJSON (List features, String name, String externalId = "", String notes = "") {
static Map createConvexHullGeoJSON (List features, String name, String externalId = "", String notes = "", String description = "") {
features = features.findAll { it.geometry != null }
List featureGeometries = features.collect { it.geometry }
Geometry geometry = GeometryUtils.getFeatureCollectionConvexHull(featureGeometries)
Expand All @@ -346,7 +346,7 @@ class ParatooProtocolConfig {
name: name,
externalId: externalId,
notes: notes,
description: "${name} (convex hull of all features)",
description: "${description?:name}",
],
features: features
]
Expand Down
121 changes: 117 additions & 4 deletions src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,9 +12,6 @@ import org.codehaus.jackson.map.ObjectMapper
import org.grails.web.converters.marshaller.json.CollectionMarshaller
import org.grails.web.converters.marshaller.json.MapMarshaller

import java.time.format.DateTimeTextProvider
import java.time.temporal.TemporalField

import static grails.async.Promises.waitAll
/**
* Tests for the ParatooService.
Expand Down Expand Up @@ -82,6 +79,7 @@ class ParatooServiceSpec extends MongoSpec implements ServiceUnitTest<ParatooSer
Service.findAll().each { it.delete() }
UserPermission.findAll().each { it.delete() }
Program.findAll().each { it.delete() }
Site.findAll().each { it.delete() }
}

def cleanup() {
Expand Down Expand Up @@ -339,7 +337,7 @@ class ParatooServiceSpec extends MongoSpec implements ServiceUnitTest<ParatooSer

and:
site.name == "SATFLB0001 - Control (100 x 100)"
site.description == "SATFLB0001 - Control (100 x 100) (convex hull of all features)"
site.description == "SATFLB0001 - Control (100 x 100)"
site.notes == "Core monitoring plot some comment"
site.type == "compound"
site.publicationStatus == "published"
Expand All @@ -350,6 +348,121 @@ class ParatooServiceSpec extends MongoSpec implements ServiceUnitTest<ParatooSer

}

void "The service will create a new site if ploy layout has been updated or use existing site" () {
setup:
String projectId = 'p1'
String orgMintedId = 'd1'
Date afterSubmissionDate = DateUtil.parseWithMilliseconds("2023-09-15T06:00:11.996Z")
Date beforeSubmissionDate = DateUtil.parseWithMilliseconds("2023-09-14T06:00:11.996Z")
new Site(
name: "SATFLB0001 - Control (100 x 100)",
siteId: "s0",
extent: [geometry: DUMMY_POLYGON],
description: "SATFLB0001 - Control (100 x 100)",
notes: "Core monitoring plot some comment",
type: "compound",
externalIds: [new ExternalId(externalId: "2", idType: ExternalId.IdType.MONITOR_PLOT_GUID)],
dateCreated: afterSubmissionDate,
lastUpdated: afterSubmissionDate
).save(flush: true)
ParatooProtocolId protocol = new ParatooProtocolId(id: "1", version: 1)
ParatooCollection collection = new ParatooCollection(
orgMintedUUID:orgMintedId,
coreProvenance: [
"system_core": "<system-core>",
"version_core": "<core-version>"
]
)
ParatooCollectionId paratooCollectionId = buildCollectionId("mintCollectionIdBasalAreaPayload","guid-3")
Map dataSet = [dataSetId:'d1', grantId:'g1', surveyId:paratooCollectionId.toMap()]
ParatooProject project = new ParatooProject(id: projectId, project: new Project(projectId: projectId, custom: [dataSets: [dataSet]]))
Map surveyData = readSurveyData('basalAreaDbhReverseLookup')
Map site

when:
Map result = service.submitCollection(collection, project)
waitAll(result.promise)
println ("finished waiting")

then:
1 * webService.doPost(*_) >> [resp: surveyData]
1 * tokenService.getAuthToken(true) >> Mock(AccessToken)
2 * projectService.updateDataSet(projectId, _) >> [status: 'ok']
0 * siteService.create(_)
1 * activityService.create({
it.startDate == "2023-09-22T00:59:47Z" && it.endDate == "2023-09-23T00:59:47Z" &&
it.plannedStartDate == "2023-09-22T00:59:47Z" && it.plannedEndDate == "2023-09-23T00:59:47Z" &&
it.externalIds[0].externalId == "d1" && it.externalIds[0].idType == ExternalId.IdType.MONITOR_MINTED_COLLECTION_ID
}) >> [activityId: '123']
1 * recordService.getAllByActivity('123') >> []
1 * settingService.getSetting('paratoo.surveyData.mapping') >> {
(["guid-3": [
"name" : "Basal Area - DBH",
"usesPlotLayout": true,
"tags" : ["survey"],
"apiEndpoint" : "basal-area-dbh-measure-surveys",
"overrides" : [
"dataModel": null,
"viewModel": null
]
]] as JSON).toString()
}
1 * userService.getCurrentUserDetails() >> [userId: userId]
1 * userService.setCurrentUser(userId)

when:
String date = DateUtil.format(new Date())
date = date.replace("Z", ".999Z")
surveyData["collections"]["basal-area-dbh-measure-survey"]["plot_visit"]["plot_layout"]["updatedAt"] = [date]
result = service.submitCollection(collection, project)

then:
1 * webService.doPost(*_) >> [resp: surveyData]
1 * tokenService.getAuthToken(true) >> Mock(AccessToken)
2 * projectService.updateDataSet(projectId, _) >> [status: 'ok']
1 * siteService.create(_) >> { site = it[0]; [siteId: 's1'] }
1 * activityService.create({
it.startDate == "2023-09-22T00:59:47Z" && it.endDate == "2023-09-23T00:59:47Z" &&
it.plannedStartDate == "2023-09-22T00:59:47Z" && it.plannedEndDate == "2023-09-23T00:59:47Z" &&
it.externalIds[0].externalId == "d1" && it.externalIds[0].idType == ExternalId.IdType.MONITOR_MINTED_COLLECTION_ID
}) >> [activityId: '123']
1 * recordService.getAllByActivity('123') >> []
1 * settingService.getSetting('paratoo.surveyData.mapping') >> {
(["guid-3": [
"name" : "Basal Area - DBH",
"usesPlotLayout": true,
"tags" : ["survey"],
"apiEndpoint" : "basal-area-dbh-measure-surveys",
"overrides" : [
"dataModel": null,
"viewModel": null
]
]] as JSON).toString()
}
1 * userService.getCurrentUserDetails() >> [userId: userId]
1 * userService.setCurrentUser(userId)
result.updateResult == [status: 'ok']
}

void "isUpdatedPlotLayout should check plot layout has been updated after site has been updated" () {
given:
def date1 = DateUtil.parseWithMilliseconds("2023-09-22T00:59:47.111Z")
def date2 = DateUtil.parseWithMilliseconds("2023-09-23T00:59:47.111Z")
def date3 = DateUtil.parseWithMilliseconds("2023-09-24T00:59:47.111Z")

when:
def result = service.isUpdatedPlotLayout(date1, date2)

then:
result

when:
result = service.isUpdatedPlotLayout(date3, date2)

then:
!result
}

private Map getProject(){
[
projectId:"p1",
Expand Down
23 changes: 20 additions & 3 deletions src/test/groovy/au/org/ala/ecodata/SiteServiceSpec.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,13 +4,12 @@ import com.mongodb.BasicDBObject
import grails.converters.JSON
import grails.test.mongodb.MongoSpec
import grails.testing.services.ServiceUnitTest
import org.grails.web.converters.marshaller.json.CollectionMarshaller

/*import grails.test.mixin.TestMixin
import grails.test.mixin.mongodb.MongoDbTestMixin*/
import org.grails.web.converters.marshaller.json.CollectionMarshaller
import org.grails.web.converters.marshaller.json.MapMarshaller
import spock.lang.Specification

import org.grails.web.converters.marshaller.json.MapMarshaller
/**
* Specification / tests for the SiteService
*/
Expand Down Expand Up @@ -288,6 +287,24 @@ class SiteServiceSpec extends MongoSpec implements ServiceUnitTest<SiteService>
}
def "Sites can be listed by externalId and sorted"() {
when:
def result
Site.withSession { session ->
result = service.create([name:'Site 1', siteId:"s1", externalIds:[new ExternalId(externalId:'e1', idType:ExternalId.IdType.MONITOR_PLOT_GUID)]])
session.flush()
result = service.create([name:'Site 2', siteId:"s2", externalIds:[new ExternalId(externalId:'e1', idType:ExternalId.IdType.MONITOR_PLOT_GUID)]])
session.flush()
}
then:
def sites = Site.findAllByExternalId(ExternalId.IdType.MONITOR_PLOT_GUID, 'e1', ['sort': "lastUpdated", 'order': "desc"])
sites.size() == 2
sites[0].name == 'Site 2'
sites[0].externalIds.size() == 1
sites.externalIds.externalId == [['e1'], ['e1']]
sites.externalIds.idType == [[ExternalId.IdType.MONITOR_PLOT_GUID], [ExternalId.IdType.MONITOR_PLOT_GUID]]
}
private Map buildExtent(source, type, coordinates, pid = '') {
return [source:source, geometry:[type:type, coordinates: coordinates, pid:pid]]
Expand Down
Loading

0 comments on commit 93ad03d

Please sign in to comment.