Merge remote-tracking branch 'origin/dev' into feature/issue994

build.gradle

@@ -20,7 +20,7 @@ plugins {
  id "com.gorylenko.gradle-git-properties" version "2.4.1"
 }
 
-version "5.0-SNAPSHOT"
+version "5.1-SNAPSHOT"
 group "au.org.ala"
 description "Ecodata"
 

grails-app/controllers/au/org/ala/ecodata/AdminController.groovy

@@ -270,9 +270,11 @@ class AdminController {
  def defaultStartDate = "2018-01-01"
  def timeZoneUTC = TimeZone.getTimeZone("UTC")
  dateFormat.setTimeZone(timeZoneUTC)
- def isMERIT = params.getBoolean('isMERIT', true)
+ Boolean isForceFetch = params.getBoolean('force', true)
+ Boolean isMERIT = params.getBoolean('isMERIT', true)
  Date startDate = params.getDate("startDate", ["yyyy", "yyyy-MM-dd"]) ?: dateFormat.parse(defaultStartDate)
- List siteIds = params.get("siteId")?.split(",")
+ List siteIds = params.get("siteId")?.split(",") ?: []
+ List projectIds = params.get("projectId")?.split(",") ?: []
  def code = 'success'
  def total = 0
  def offset = 0
@@ -281,8 +283,14 @@ class AdminController {
  List<String> defaultFids = metadataService.getSpatialLayerIdsToIntersect()
  log.debug("Number of fids to intersect: ${defaultFids.size()}; they are - ${defaultFids}")
  def totalSites
- List projectIds = []
- if (siteIds) {
+ if (projectIds) {
+ projectIds.each {
+ siteIds.addAll(siteService.findAllSiteIdsForProject(it))
+ }
+
+ totalSites = siteIds.size()
+ }
+ else if (siteIds) {
  totalSites = siteIds.size()
  }
  else if (isMERIT) {
@@ -325,13 +333,20 @@ class AdminController {
  log.info("${total+1} or ${(total+1)*100/totalSites} % sites updated in db..")
  }
 
- if (!site.projects || !site.extent) {
- log.debug("Ignoring site ${site.siteId} due to no associated projects or no extent")
+ if (!site.extent) {
+ log.debug("Ignoring site ${site.siteId} due to no extent")
  return
  }
- def projectsOfSite = site.projects
- List hubIds = projectService.findHubIdOfProjects(projectsOfSite)
+ // management unit site does not have any projects
+ def projectsOfSite = site.projects ?: []
+ List hubIds = projectService.findHubIdFromProjectsOrCurrentHub(projectsOfSite)
  def fids = hubIds.size() == 1 ? metadataService.getSpatialLayerIdsToIntersect(hubIds[0]) : defaultFids
+
+ if (!isForceFetch && siteService.areIntersectionCalculatedForAllLayers(site)) {
+ log.debug("Skipping site ${site.siteId} as all layers are already calculated and force fetch is not enabled - $isForceFetch")
+ return // Skip if all layers are already calculated
+ }
+
  siteService.populateLocationMetadataForSite(site, fids)
  endInterimTime = System.currentTimeMillis()
  log.debug("Time taken to update metadata ${site.siteId}: ${endInterimTime - startInterimTime} ms")
@@ -345,7 +360,7 @@ class AdminController {
  }
  }
  catch (Exception e) {
- log.error("Unable to complete the operation ", e)
+ log.error("Unable to complete the operation for siteId - ${site.siteId} ", e)
  code = "error"
  }
  }

grails-app/jobs/au/org/ala/ecodata/ImportSciStarterJob.groovy

@@ -10,7 +10,7 @@ class ImportSciStarterJob {
  static triggers = {
  Boolean enabled = grailsApplication.config.getProperty("sciStarter.importEnabled", Boolean, true)
  if (enabled) {
- cron name: "every sunday", cronExpression: "0 0 0 ? * 1/7 *"
+ cron name: "every sunday", cronExpression: grailsApplication.config.getProperty("scistarter.cronSchedule", "0 0 0 ? * 1/7 *")
  }
  }
 

grails-app/services/au/org/ala/ecodata/MetadataService.groovy

@@ -490,7 +490,7 @@ class MetadataService {
  * @return
  */
  List<String> getSpatialLayerIdsToIntersectForProjects(List projectIds = []) {
- List hubIds = projectService.findHubIdOfProjects(projectIds)
+ List hubIds = projectService.findHubIdFromProjectsOrCurrentHub(projectIds)
  hubIds.size() == 1 ? getSpatialLayerIdsToIntersect(hubIds[0]) : getSpatialLayerIdsToIntersect()
  }
 

grails-app/services/au/org/ala/ecodata/ProjectService.groovy

@@ -51,6 +51,7 @@ class ProjectService {
  ActivityFormService activityFormService
  RecordService recordService
  LockService lockService
+ HubService hubService
 
  /* def getCommonService() {
  grailsApplication.mainContext.commonService
@@ -1262,4 +1263,19 @@ class ProjectService {
  }
  }
 
+ /**
+ * Find hubs from project or use hubId query parameter
+ * @param projects
+ * @return
+ */
+ def findHubIdFromProjectsOrCurrentHub (List projects) {
+ if (projects) {
+ return findHubIdOfProjects(projects)
+ }
+ else {
+ def currentHub = hubService.getCurrentHub()
+ return currentHub ? [currentHub.hubId] : []
+ }
+ }
+
 }

grails-app/services/au/org/ala/ecodata/SiteService.groovy

@@ -258,23 +258,28 @@ class SiteService {
  def getSimpleProjectArea(projectSiteId) {
  def threshold = grailsApplication.config.getProperty('biocollect.projectArea.simplificationThreshold', Integer, 10000)
  def tolerance = grailsApplication.config.getProperty('biocollect.projectArea.simplificationTolerance', Double, 0.0001)
- log.info("Threshhold ${threshold} Tolerance ${tolerance}")
 
  def site = get(projectSiteId, [SiteService.FLAT, SiteService.INDEXING])
 
- if (site != null) {
- def projectArea = geometryAsGeoJson(site)
-
- if (projectArea?.coordinates != null) {
- def coordsSize = projectArea.coordinates.flatten().size()
- if (coordsSize > threshold) {
- site.geoIndex = GeometryUtils.simplify(projectArea, tolerance)
- } else {
- site.geoIndex = projectArea
+ try {
+ if (site != null) {
+ def projectArea = geometryAsGeoJson(site)
+
+ if (projectArea?.coordinates != null) {
+ def coordsSize = projectArea.coordinates.flatten().size()
+ if (coordsSize > threshold) {
+ site.geoIndex = GeometryUtils.simplify(projectArea, tolerance)
+ } else {
+ site.geoIndex = projectArea
+ }
  }
  }
+ } catch (Exception e) {
+ log.info("Unable to get simplified project area geometry (site ${site.siteId})", e)
  }
 
+ // remove extent to not avoid total fields limit in ES
+ site?.remove('extent')
  site
  }
 
@@ -731,7 +736,7 @@ class SiteService {
  */
  def mergeIntersectionsArea(Map site, Map intersectionsAreaByFacets) {
  Map geometry = site.extent.geometry
- List hubs = projectService.findHubIdOfProjects(site.projects)
+ List hubs = projectService.findHubIdFromProjectsOrCurrentHub(site.projects)
  String hubId = hubs?.size() == 1 ? hubs[0] : null
  Map existingIntersectionsArea = geometry[SpatialService.INTERSECTION_AREA] = geometry[SpatialService.INTERSECTION_AREA] ?: [:]
  intersectionsAreaByFacets?.each { String layer, Map nameAndValue ->
@@ -1085,4 +1090,18 @@ class SiteService {
  List filterSitesByPurposeIsPlanning (List<Map> sites) {
  sites?.findAll { getPurpose(it) == Site.PLANNING_SITE_CODE }
  }
+
+ /**
+ * Check if the intersection area is calculated for all provided layers
+ * @param site
+ * @param fids
+ * @return
+ */
+ Boolean areIntersectionCalculatedForAllLayers(def site){
+ List fids = metadataService.getGeographicConfig().checkForBoundaryIntersectionInLayers
+ fids?.every { fid ->
+ String group = metadataService.getGeographicFacetConfig(fid, null)?.name
+ site.extent?.geometry?[SpatialService.INTERSECTION_AREA]?[group]?[fid] != null
+ }
+ }
 }

grails-app/services/au/org/ala/ecodata/SpatialService.groovy

@@ -4,10 +4,7 @@ import grails.core.GrailsApplication
 import grails.plugin.cache.Cacheable
 import groovy.json.JsonParserType
 import groovy.json.JsonSlurper
-import org.locationtech.jts.geom.Coordinate
-import org.locationtech.jts.geom.Geometry
-import org.locationtech.jts.geom.GeometryCollection
-import org.locationtech.jts.geom.GeometryFactory
+import org.locationtech.jts.geom.*
 import org.locationtech.jts.io.WKTReader
 
 import static ParatooService.deepCopy
@@ -68,12 +65,22 @@ class SpatialService {
  } else {
  geo = GeometryUtils.geoJsonMapToGeometry (geoJson)
  GeometryCollection geometryCollection = (GeometryCollection)geo
+ Geometry correctGeometry = geometryCollection
  if(!geometryCollection.isValid()) {
- geometryCollection = geometryCollection.buffer(0)
+ correctGeometry = geometryCollection.buffer(0)
  }
 
- Geometry convexHullGeometry = geometryCollection.union().convexHull()
- wkt = convexHullGeometry.toText()
+ Geometry convexHullGeometry
+ try {
+ convexHullGeometry = correctGeometry.union().convexHull()
+ }
+ catch (TopologyException e) {
+ log.error("Error creating convex hull for geometry collection")
+ convexHullGeometry = correctGeometry.buffer(0).union().convexHull()
+ }
+ finally {
+ wkt = convexHullGeometry?.toText()
+ }
  }
 
  String url = grailsApplication.config.getProperty('spatial.baseUrl')+WKT_INTERSECT_URL_PREFIX

src/main/groovy/au/org/ala/ecodata/GeometryUtils.groovy

@@ -18,6 +18,8 @@ import org.opengis.referencing.crs.CoordinateReferenceSystem
 import org.opengis.referencing.operation.MathTransform
 
 import java.awt.geom.Point2D
+import java.nio.charset.StandardCharsets
+
 /**
  * Helper class for working with site geometry.
  */
@@ -29,10 +31,13 @@ class GeometryUtils {
 
  static Map wktToGeoJson(String wkt, int decimals = 20) {
  WKTReader wktReader = new WKTReader()
- Geometry geom = wktReader.read(wkt)
- String geoJSON = new GeometryJSON(decimals).toString(geom)
+ ByteArrayOutputStream outputStream = new ByteArrayOutputStream()
+ OutputStreamWriter writer = new OutputStreamWriter(outputStream, StandardCharsets.UTF_8)
+ def reader = new StringReader(wkt)
+ Geometry geom = wktReader.read(reader)
+ new GeometryJSON(decimals).write(geom, writer)
  ObjectMapper mapper = new ObjectMapper()
- return mapper.readValue(geoJSON, Map)
+ return mapper.readValue(outputStream.toByteArray(), Map)
  }
 
  static String wktToMultiPolygonWkt(String wkt) {

src/main/groovy/au/org/ala/ecodata/reporting/ProjectXlsExporter.groovy

@@ -340,6 +340,19 @@ class ProjectXlsExporter extends ProjectExporter {
  else
  log.error ("No facet config found for layer $layer.")
  }
+
+ if (project.geographicInfo) {
+ // load from manually assigned electorates/states
+ if (!project.containsKey(getPropertyNameForFacet("elect"))) {
+ project[getPropertyNameForFacet("elect")] = project.geographicInfo.primaryElectorate
+ project[getPropertyNameForFacet("elect","other")] = project.geographicInfo.otherElectorates?.join("; ")
+ }
+
+ if (!project.containsKey(getPropertyNameForFacet("state"))) {
+ project[getPropertyNameForFacet("state")] = project.geographicInfo.primaryState
+ project[getPropertyNameForFacet("state","other")] = project.geographicInfo.otherStates?.join("; ")
+ }
+ }
  }
 
  private addProjectGeo(Map project) {

src/test/groovy/au/org/ala/ecodata/SiteServiceSpec.groovy

@@ -194,7 +194,7 @@ class SiteServiceSpec extends MongoSpec implements ServiceUnitTest<SiteService>
  def "New sites without a centroid should have one assigned"() {
  when:
  def result
- projectService.findHubIdOfProjects(_) >> []
+ projectService.findHubIdFromProjectsOrCurrentHub(_) >> []
  Site.withSession { session ->
  result = service.create([name: 'Site 1', extent: [source: 'pid', geometry: [type: 'pid', pid: 'cl123']]])
  session.flush()
@@ -310,7 +310,7 @@ class SiteServiceSpec extends MongoSpec implements ServiceUnitTest<SiteService>
 
  def "The site area is calculated from the FeatureCollection for a compound site"() {
  setup:
- projectService.findHubIdOfProjects(_) >> []
+ projectService.findHubIdFromProjectsOrCurrentHub(_) >> []
  def coordinates = [[148.260498046875, -37.26530995561874], [148.260498046875, -37.26531995561874], [148.310693359375, -37.26531995561874], [148.310693359375, -37.26531995561874], [148.260498046875, -37.26530995561874]]
  def extent = buildExtent('drawn', 'Polygon', coordinates)
  Map site = [type: Site.TYPE_COMPOUND, extent: extent, features: [
@@ -362,8 +362,7 @@ class SiteServiceSpec extends MongoSpec implements ServiceUnitTest<SiteService>
  def newSite = service.create([name:'Site 1', extent: extent])
 
  then:
- def site = service.getSimpleProjectArea(newSite.siteId)
- site.geoIndex != null
+ service.getSimpleProjectArea(newSite.siteId) != null
  }
 
  private Map buildExtent(source, type, coordinates, pid = '') {