AtlasOfLivingAustralia/fieldcapture#3292

- fixed issue with large GeometryCollection object intersection with spatial layer failing - added cache to pid intersection - added siteId parameter to controller action
AtlasOfLivingAustralia · Sep 18, 2024 · 8a93bab · 8a93bab
1 parent 11b6dcc
commit 8a93bab
Show file tree

Hide file tree

Showing 3 changed files with 52 additions and 31 deletions.
diff --git a/grails-app/conf/ecodata-ehcache.xml b/grails-app/conf/ecodata-ehcache.xml
@@ -38,6 +38,20 @@
  <heap unit="entries">100</heap>
  </resources>
  </cache>
+ <cache alias="spatialPidFidIntersection">
+
+ <key-type>java.io.Serializable</key-type>
+ <value-type>java.io.Serializable</value-type>
+
+ <expiry>
+ <tti unit="days">365</tti>
+ </expiry>
+ <resources>
+ <heap unit="entries">200</heap>
+
+ <disk unit="MB" persistent="true">500</disk>
+ </resources>
+ </cache>
  <cache alias="spatialPidObjectGeometry">
 
  <key-type>java.io.Serializable</key-type>

diff --git a/grails-app/controllers/au/org/ala/ecodata/AdminController.groovy b/grails-app/controllers/au/org/ala/ecodata/AdminController.groovy
@@ -272,6 +272,7 @@ class AdminController {
  dateFormat.setTimeZone(timeZoneUTC)
  def isMERIT = params.getBoolean('isMERIT', true)
  Date startDate = params.getDate("startDate", ["yyyy", "yyyy-MM-dd"]) ?: dateFormat.parse(defaultStartDate)
+ List siteIds = params.get("siteId")?.split(",")
  def code = 'success'
  def total = 0
  def offset = 0
@@ -281,7 +282,10 @@ class AdminController {
  log.debug("Number of fids to intersect: ${defaultFids.size()}; they are - ${defaultFids}")
  def totalSites
  List projectIds = []
- if (isMERIT) {
+ if (siteIds) {
+ totalSites = siteIds.size()
+ }
+ else if (isMERIT) {
  projectIds = projectService.getAllMERITProjectIds()
  totalSites = Site.countByStatusAndProjectsInListAndDateCreatedGreaterThan('active', projectIds, startDate)
  }
@@ -293,7 +297,12 @@ class AdminController {
  while (count == batchSize) {
  batchStartTime = startInterimTime = System.currentTimeMillis()
  def sites
- if (isMERIT) {
+ if (siteIds) {
+ sites = Site.findAllBySiteIdInList(siteIds, [offset: offset, max: batchSize, sort: "siteId", order: "asc"]).collect {
+ siteService.toMap(it, 'flat')
+ }
+ }
+ else if (isMERIT) {
  sites = Site.findAllByProjectsInListAndStatusAndDateCreatedGreaterThan(projectIds, 'active', startDate, [offset: offset, max: batchSize, sort: "siteId", order: "asc"]).collect {
  siteService.toMap(it, 'flat')
  }

diff --git a/grails-app/services/au/org/ala/ecodata/SpatialService.groovy b/grails-app/services/au/org/ala/ecodata/SpatialService.groovy
@@ -47,11 +47,25 @@ class SpatialService {
  * supplied geometry
  */
  Map<String,?> intersectGeometry(Map geoJson, List<String> fieldIds = null) {
- int length = geoJson?.toString().size()
+ // We are using a WKT string instead of geojson as the spatial portal validates geojson - using
+ // WKT allows us to get away with self intersecting polygons that users occasionally draw.
+ String wkt
  int threshold = grailsApplication.config.getProperty('spatial.geoJsonEnvelopeConversionThreshold', Integer)
- Geometry geo = GeometryUtils.geoJsonMapToGeometry (geoJson)
- if(length > threshold){
- geoJson = GeometryUtils.geometryToGeoJsonMap (geo.getEnvelope())
+ Geometry geo
+ // Ignore threshold check for GeometryCollection collection
+ if (geo.geometryType != 'GeometryCollection') {
+ int length = geoJson?.toString().size()
+ if (length > threshold) {
+ geoJson = GeometryUtils.geometryToGeoJsonMap(geo.getEnvelope())
+ }
+
+ geo = GeometryUtils.geoJsonMapToGeometry (geoJson)
+ wkt = geo.toText()
+ } else {
+ geo = GeometryUtils.geoJsonMapToGeometry (geoJson)
+ GeometryCollection geometryCollection = (GeometryCollection)geo
+ Geometry convexHullGeometry = geometryCollection.union().convexHull()
+ wkt = convexHullGeometry.toText()
  }
 
  String url = grailsApplication.config.getProperty('spatial.baseUrl')+WKT_INTERSECT_URL_PREFIX
@@ -60,36 +74,16 @@ class SpatialService {
  }
 
  long start = System.currentTimeMillis()
- // We are using a WKT string instead of geojson as the spatial portal validates geojson - using
- // WKT allows us to get away with self intersecting polygons that users occasionally draw.
- String wkt = geo.toText()
  long end = System.currentTimeMillis()
  log.info("Time taken to convert geojson to wkt: ${end-start}ms")
 
 
  Map result = [:]
  fieldIds.each { fid ->
  start = end
- Map response
- if (geo.geometryType == 'GeometryCollection') {
- Map<String, Map> geometryCollectionIntersections = [:]
- GeometryCollection geometryCollection = (GeometryCollection)geo
- for (int i=0; i<geometryCollection.numGeometries; i++) {
- Geometry geometryN = geometryCollection.getGeometryN(i)
- String wktGeometryN = geometryN.toText()
- response = webService.doPost(url+fid, wktGeometryN)
- if (response.resp && !response.error) {
- response.resp?.each {geometryCollectionIntersections[it.pid] = it }
- }
- }
-
- result[fid] = geometryCollectionIntersections.values()?.toList()
- }
- else {
- response = webService.doPost(url+fid, wkt)
- if (response.resp && !response.error) {
- result[fid] = response.resp
- }
+ Map response = webService.doPost(url+fid, wkt)
+ if (response.resp && !response.error) {
+ result[fid] = response.resp
  }
  end = System.currentTimeMillis()
  log.info("Time taken to intersect with layer $fid: ${end-start}ms")
@@ -103,7 +97,6 @@ class SpatialService {
  geographicFacets = [:].withDefault{[]}
  GeometryCollection geometryCollection = (GeometryCollection)geo
  for (int i=0; i<geometryCollection.numGeometries; i++) {
-
  def (filtered, intersectionArea) = filterOutObjectsInBoundary(result, geometryCollection.getGeometryN(i))
  start = end
  Map geographicFacetsForGeometry = convertResponsesToGeographicFacets(filtered)
@@ -152,7 +145,7 @@ class SpatialService {
  result[fid] = lookupTable[pidFid][pid][fid].collect{[name:it]}
  }
  else {
- Object response = webService.getJson(url+fid+"/"+pid)
+ def response = getPidFidIntersection(url, fid, pid)
  if (response instanceof List) {
  result[fid] = response
  }
@@ -167,6 +160,11 @@ class SpatialService {
  geographicFacets
  }
 
+ @Cacheable(value = "spatialPidFidIntersection")
+ def getPidFidIntersection(String url, String fid, String pid) {
+ webService.getJson(url + fid + "/" + pid)
+ }
+
  private List filterOutObjectsInBoundary(Map response, Map mainObjectGeoJson) {
  Geometry mainGeometry = GeometryUtils.geoJsonMapToGeometry(mainObjectGeoJson)
  filterOutObjectsInBoundary(response, mainGeometry)