diff --git a/build.gradle b/build.gradle index 09bbe4ed1..2c18d5340 100644 --- a/build.gradle +++ b/build.gradle @@ -20,7 +20,7 @@ plugins { id "com.gorylenko.gradle-git-properties" version "2.4.1" } -version "5.0-SNAPSHOT" +version "5.1-SNAPSHOT" group "au.org.ala" description "Ecodata" diff --git a/grails-app/controllers/au/org/ala/ecodata/AdminController.groovy b/grails-app/controllers/au/org/ala/ecodata/AdminController.groovy index 67a21b83a..5432ccbdd 100644 --- a/grails-app/controllers/au/org/ala/ecodata/AdminController.groovy +++ b/grails-app/controllers/au/org/ala/ecodata/AdminController.groovy @@ -270,9 +270,11 @@ class AdminController { def defaultStartDate = "2018-01-01" def timeZoneUTC = TimeZone.getTimeZone("UTC") dateFormat.setTimeZone(timeZoneUTC) - def isMERIT = params.getBoolean('isMERIT', true) + Boolean isForceFetch = params.getBoolean('force', true) + Boolean isMERIT = params.getBoolean('isMERIT', true) Date startDate = params.getDate("startDate", ["yyyy", "yyyy-MM-dd"]) ?: dateFormat.parse(defaultStartDate) - List siteIds = params.get("siteId")?.split(",") + List siteIds = params.get("siteId")?.split(",") ?: [] + List projectIds = params.get("projectId")?.split(",") ?: [] def code = 'success' def total = 0 def offset = 0 @@ -281,8 +283,14 @@ class AdminController { List defaultFids = metadataService.getSpatialLayerIdsToIntersect() log.debug("Number of fids to intersect: ${defaultFids.size()}; they are - ${defaultFids}") def totalSites - List projectIds = [] - if (siteIds) { + if (projectIds) { + projectIds.each { + siteIds.addAll(siteService.findAllSiteIdsForProject(it)) + } + + totalSites = siteIds.size() + } + else if (siteIds) { totalSites = siteIds.size() } else if (isMERIT) { @@ -325,13 +333,20 @@ class AdminController { log.info("${total+1} or ${(total+1)*100/totalSites} % sites updated in db..") } - if (!site.projects || !site.extent) { - log.debug("Ignoring site ${site.siteId} due to no associated projects or no extent") + if (!site.extent) { + log.debug("Ignoring site ${site.siteId} due to no extent") return } - def projectsOfSite = site.projects - List hubIds = projectService.findHubIdOfProjects(projectsOfSite) + // management unit site does not have any projects + def projectsOfSite = site.projects ?: [] + List hubIds = projectService.findHubIdFromProjectsOrCurrentHub(projectsOfSite) def fids = hubIds.size() == 1 ? metadataService.getSpatialLayerIdsToIntersect(hubIds[0]) : defaultFids + + if (!isForceFetch && siteService.areIntersectionCalculatedForAllLayers(site)) { + log.debug("Skipping site ${site.siteId} as all layers are already calculated and force fetch is not enabled - $isForceFetch") + return // Skip if all layers are already calculated + } + siteService.populateLocationMetadataForSite(site, fids) endInterimTime = System.currentTimeMillis() log.debug("Time taken to update metadata ${site.siteId}: ${endInterimTime - startInterimTime} ms") @@ -345,7 +360,7 @@ class AdminController { } } catch (Exception e) { - log.error("Unable to complete the operation ", e) + log.error("Unable to complete the operation for siteId - ${site.siteId} ", e) code = "error" } } diff --git a/grails-app/jobs/au/org/ala/ecodata/ImportSciStarterJob.groovy b/grails-app/jobs/au/org/ala/ecodata/ImportSciStarterJob.groovy index eec02b3c1..b861010cd 100644 --- a/grails-app/jobs/au/org/ala/ecodata/ImportSciStarterJob.groovy +++ b/grails-app/jobs/au/org/ala/ecodata/ImportSciStarterJob.groovy @@ -10,7 +10,7 @@ class ImportSciStarterJob { static triggers = { Boolean enabled = grailsApplication.config.getProperty("sciStarter.importEnabled", Boolean, true) if (enabled) { - cron name: "every sunday", cronExpression: "0 0 0 ? * 1/7 *" + cron name: "every sunday", cronExpression: grailsApplication.config.getProperty("scistarter.cronSchedule", "0 0 0 ? * 1/7 *") } } diff --git a/grails-app/services/au/org/ala/ecodata/MetadataService.groovy b/grails-app/services/au/org/ala/ecodata/MetadataService.groovy index 02f9885cb..eb95b00ef 100644 --- a/grails-app/services/au/org/ala/ecodata/MetadataService.groovy +++ b/grails-app/services/au/org/ala/ecodata/MetadataService.groovy @@ -490,7 +490,7 @@ class MetadataService { * @return */ List getSpatialLayerIdsToIntersectForProjects(List projectIds = []) { - List hubIds = projectService.findHubIdOfProjects(projectIds) + List hubIds = projectService.findHubIdFromProjectsOrCurrentHub(projectIds) hubIds.size() == 1 ? getSpatialLayerIdsToIntersect(hubIds[0]) : getSpatialLayerIdsToIntersect() } diff --git a/grails-app/services/au/org/ala/ecodata/ProjectService.groovy b/grails-app/services/au/org/ala/ecodata/ProjectService.groovy index bcff4e56a..84e647b98 100644 --- a/grails-app/services/au/org/ala/ecodata/ProjectService.groovy +++ b/grails-app/services/au/org/ala/ecodata/ProjectService.groovy @@ -51,6 +51,7 @@ class ProjectService { ActivityFormService activityFormService RecordService recordService LockService lockService + HubService hubService /* def getCommonService() { grailsApplication.mainContext.commonService @@ -1262,4 +1263,19 @@ class ProjectService { } } + /** + * Find hubs from project or use hubId query parameter + * @param projects + * @return + */ + def findHubIdFromProjectsOrCurrentHub (List projects) { + if (projects) { + return findHubIdOfProjects(projects) + } + else { + def currentHub = hubService.getCurrentHub() + return currentHub ? [currentHub.hubId] : [] + } + } + } \ No newline at end of file diff --git a/grails-app/services/au/org/ala/ecodata/SiteService.groovy b/grails-app/services/au/org/ala/ecodata/SiteService.groovy index ccc7bb0fb..7e656c58c 100644 --- a/grails-app/services/au/org/ala/ecodata/SiteService.groovy +++ b/grails-app/services/au/org/ala/ecodata/SiteService.groovy @@ -258,23 +258,28 @@ class SiteService { def getSimpleProjectArea(projectSiteId) { def threshold = grailsApplication.config.getProperty('biocollect.projectArea.simplificationThreshold', Integer, 10000) def tolerance = grailsApplication.config.getProperty('biocollect.projectArea.simplificationTolerance', Double, 0.0001) - log.info("Threshhold ${threshold} Tolerance ${tolerance}") def site = get(projectSiteId, [SiteService.FLAT, SiteService.INDEXING]) - if (site != null) { - def projectArea = geometryAsGeoJson(site) - - if (projectArea?.coordinates != null) { - def coordsSize = projectArea.coordinates.flatten().size() - if (coordsSize > threshold) { - site.geoIndex = GeometryUtils.simplify(projectArea, tolerance) - } else { - site.geoIndex = projectArea + try { + if (site != null) { + def projectArea = geometryAsGeoJson(site) + + if (projectArea?.coordinates != null) { + def coordsSize = projectArea.coordinates.flatten().size() + if (coordsSize > threshold) { + site.geoIndex = GeometryUtils.simplify(projectArea, tolerance) + } else { + site.geoIndex = projectArea + } } } + } catch (Exception e) { + log.info("Unable to get simplified project area geometry (site ${site.siteId})", e) } + // remove extent to not avoid total fields limit in ES + site?.remove('extent') site } @@ -731,7 +736,7 @@ class SiteService { */ def mergeIntersectionsArea(Map site, Map intersectionsAreaByFacets) { Map geometry = site.extent.geometry - List hubs = projectService.findHubIdOfProjects(site.projects) + List hubs = projectService.findHubIdFromProjectsOrCurrentHub(site.projects) String hubId = hubs?.size() == 1 ? hubs[0] : null Map existingIntersectionsArea = geometry[SpatialService.INTERSECTION_AREA] = geometry[SpatialService.INTERSECTION_AREA] ?: [:] intersectionsAreaByFacets?.each { String layer, Map nameAndValue -> @@ -1085,4 +1090,18 @@ class SiteService { List filterSitesByPurposeIsPlanning (List sites) { sites?.findAll { getPurpose(it) == Site.PLANNING_SITE_CODE } } + + /** + * Check if the intersection area is calculated for all provided layers + * @param site + * @param fids + * @return + */ + Boolean areIntersectionCalculatedForAllLayers(def site){ + List fids = metadataService.getGeographicConfig().checkForBoundaryIntersectionInLayers + fids?.every { fid -> + String group = metadataService.getGeographicFacetConfig(fid, null)?.name + site.extent?.geometry?[SpatialService.INTERSECTION_AREA]?[group]?[fid] != null + } + } } diff --git a/grails-app/services/au/org/ala/ecodata/SpatialService.groovy b/grails-app/services/au/org/ala/ecodata/SpatialService.groovy index eb4146726..e48b1af84 100644 --- a/grails-app/services/au/org/ala/ecodata/SpatialService.groovy +++ b/grails-app/services/au/org/ala/ecodata/SpatialService.groovy @@ -4,10 +4,7 @@ import grails.core.GrailsApplication import grails.plugin.cache.Cacheable import groovy.json.JsonParserType import groovy.json.JsonSlurper -import org.locationtech.jts.geom.Coordinate -import org.locationtech.jts.geom.Geometry -import org.locationtech.jts.geom.GeometryCollection -import org.locationtech.jts.geom.GeometryFactory +import org.locationtech.jts.geom.* import org.locationtech.jts.io.WKTReader import static ParatooService.deepCopy @@ -68,12 +65,22 @@ class SpatialService { } else { geo = GeometryUtils.geoJsonMapToGeometry (geoJson) GeometryCollection geometryCollection = (GeometryCollection)geo + Geometry correctGeometry = geometryCollection if(!geometryCollection.isValid()) { - geometryCollection = geometryCollection.buffer(0) + correctGeometry = geometryCollection.buffer(0) } - Geometry convexHullGeometry = geometryCollection.union().convexHull() - wkt = convexHullGeometry.toText() + Geometry convexHullGeometry + try { + convexHullGeometry = correctGeometry.union().convexHull() + } + catch (TopologyException e) { + log.error("Error creating convex hull for geometry collection") + convexHullGeometry = correctGeometry.buffer(0).union().convexHull() + } + finally { + wkt = convexHullGeometry?.toText() + } } String url = grailsApplication.config.getProperty('spatial.baseUrl')+WKT_INTERSECT_URL_PREFIX diff --git a/src/main/groovy/au/org/ala/ecodata/GeometryUtils.groovy b/src/main/groovy/au/org/ala/ecodata/GeometryUtils.groovy index 0d73edeee..bda514e80 100644 --- a/src/main/groovy/au/org/ala/ecodata/GeometryUtils.groovy +++ b/src/main/groovy/au/org/ala/ecodata/GeometryUtils.groovy @@ -18,6 +18,8 @@ import org.opengis.referencing.crs.CoordinateReferenceSystem import org.opengis.referencing.operation.MathTransform import java.awt.geom.Point2D +import java.nio.charset.StandardCharsets + /** * Helper class for working with site geometry. */ @@ -29,10 +31,13 @@ class GeometryUtils { static Map wktToGeoJson(String wkt, int decimals = 20) { WKTReader wktReader = new WKTReader() - Geometry geom = wktReader.read(wkt) - String geoJSON = new GeometryJSON(decimals).toString(geom) + ByteArrayOutputStream outputStream = new ByteArrayOutputStream() + OutputStreamWriter writer = new OutputStreamWriter(outputStream, StandardCharsets.UTF_8) + def reader = new StringReader(wkt) + Geometry geom = wktReader.read(reader) + new GeometryJSON(decimals).write(geom, writer) ObjectMapper mapper = new ObjectMapper() - return mapper.readValue(geoJSON, Map) + return mapper.readValue(outputStream.toByteArray(), Map) } static String wktToMultiPolygonWkt(String wkt) { diff --git a/src/main/groovy/au/org/ala/ecodata/reporting/ProjectXlsExporter.groovy b/src/main/groovy/au/org/ala/ecodata/reporting/ProjectXlsExporter.groovy index 07fd28b7e..0c2b060b4 100644 --- a/src/main/groovy/au/org/ala/ecodata/reporting/ProjectXlsExporter.groovy +++ b/src/main/groovy/au/org/ala/ecodata/reporting/ProjectXlsExporter.groovy @@ -340,6 +340,19 @@ class ProjectXlsExporter extends ProjectExporter { else log.error ("No facet config found for layer $layer.") } + + if (project.geographicInfo) { + // load from manually assigned electorates/states + if (!project.containsKey(getPropertyNameForFacet("elect"))) { + project[getPropertyNameForFacet("elect")] = project.geographicInfo.primaryElectorate + project[getPropertyNameForFacet("elect","other")] = project.geographicInfo.otherElectorates?.join("; ") + } + + if (!project.containsKey(getPropertyNameForFacet("state"))) { + project[getPropertyNameForFacet("state")] = project.geographicInfo.primaryState + project[getPropertyNameForFacet("state","other")] = project.geographicInfo.otherStates?.join("; ") + } + } } private addProjectGeo(Map project) { diff --git a/src/test/groovy/au/org/ala/ecodata/SiteServiceSpec.groovy b/src/test/groovy/au/org/ala/ecodata/SiteServiceSpec.groovy index 96a5e6c06..8639be1d5 100644 --- a/src/test/groovy/au/org/ala/ecodata/SiteServiceSpec.groovy +++ b/src/test/groovy/au/org/ala/ecodata/SiteServiceSpec.groovy @@ -194,7 +194,7 @@ class SiteServiceSpec extends MongoSpec implements ServiceUnitTest def "New sites without a centroid should have one assigned"() { when: def result - projectService.findHubIdOfProjects(_) >> [] + projectService.findHubIdFromProjectsOrCurrentHub(_) >> [] Site.withSession { session -> result = service.create([name: 'Site 1', extent: [source: 'pid', geometry: [type: 'pid', pid: 'cl123']]]) session.flush() @@ -310,7 +310,7 @@ class SiteServiceSpec extends MongoSpec implements ServiceUnitTest def "The site area is calculated from the FeatureCollection for a compound site"() { setup: - projectService.findHubIdOfProjects(_) >> [] + projectService.findHubIdFromProjectsOrCurrentHub(_) >> [] def coordinates = [[148.260498046875, -37.26530995561874], [148.260498046875, -37.26531995561874], [148.310693359375, -37.26531995561874], [148.310693359375, -37.26531995561874], [148.260498046875, -37.26530995561874]] def extent = buildExtent('drawn', 'Polygon', coordinates) Map site = [type: Site.TYPE_COMPOUND, extent: extent, features: [ @@ -362,8 +362,7 @@ class SiteServiceSpec extends MongoSpec implements ServiceUnitTest def newSite = service.create([name:'Site 1', extent: extent]) then: - def site = service.getSimpleProjectArea(newSite.siteId) - site.geoIndex != null + service.getSimpleProjectArea(newSite.siteId) != null } private Map buildExtent(source, type, coordinates, pid = '') {