From 94bec0a654f134c027e10084982e41b0657de195 Mon Sep 17 00:00:00 2001 From: temi Date: Wed, 25 Sep 2024 16:27:03 +1000 Subject: [PATCH 01/19] AtlasOfLivingAustralia/fieldcapture#3292 - fixed type cast exception --- grails-app/services/au/org/ala/ecodata/SpatialService.groovy | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/grails-app/services/au/org/ala/ecodata/SpatialService.groovy b/grails-app/services/au/org/ala/ecodata/SpatialService.groovy index eb4146726..82e3391a1 100644 --- a/grails-app/services/au/org/ala/ecodata/SpatialService.groovy +++ b/grails-app/services/au/org/ala/ecodata/SpatialService.groovy @@ -68,11 +68,12 @@ class SpatialService { } else { geo = GeometryUtils.geoJsonMapToGeometry (geoJson) GeometryCollection geometryCollection = (GeometryCollection)geo + Geometry correctGeometry = geometryCollection if(!geometryCollection.isValid()) { - geometryCollection = geometryCollection.buffer(0) + correctGeometry = geometryCollection.buffer(0) } - Geometry convexHullGeometry = geometryCollection.union().convexHull() + Geometry convexHullGeometry = correctGeometry.union().convexHull() wkt = convexHullGeometry.toText() } From 97a23dedc5e075d242d6376ab2b46f7acd77d5d7 Mon Sep 17 00:00:00 2001 From: Jack Brinkman Date: Wed, 25 Sep 2024 16:35:19 +1000 Subject: [PATCH 02/19] Fixed 'then' condition of getSimpleProjectArea test --- src/test/groovy/au/org/ala/ecodata/SiteServiceSpec.groovy | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/test/groovy/au/org/ala/ecodata/SiteServiceSpec.groovy b/src/test/groovy/au/org/ala/ecodata/SiteServiceSpec.groovy index 96a5e6c06..d9d128d91 100644 --- a/src/test/groovy/au/org/ala/ecodata/SiteServiceSpec.groovy +++ b/src/test/groovy/au/org/ala/ecodata/SiteServiceSpec.groovy @@ -362,8 +362,7 @@ class SiteServiceSpec extends MongoSpec implements ServiceUnitTest def newSite = service.create([name:'Site 1', extent: extent]) then: - def site = service.getSimpleProjectArea(newSite.siteId) - site.geoIndex != null + service.getSimpleProjectArea(newSite.siteId) != null } private Map buildExtent(source, type, coordinates, pid = '') { From 76c9a3d276d91f09872af7ffb6ebd4a24177d026 Mon Sep 17 00:00:00 2001 From: temi Date: Fri, 27 Sep 2024 09:18:29 +1000 Subject: [PATCH 03/19] AtlasOfLivingAustralia/fieldcapture#3292 - update sites of a list of projects - retry convex hull if first attempt throws exception --- .../au/org/ala/ecodata/AdminController.groovy | 13 ++++++++++--- .../au/org/ala/ecodata/SpatialService.groovy | 18 ++++++++++++------ 2 files changed, 22 insertions(+), 9 deletions(-) diff --git a/grails-app/controllers/au/org/ala/ecodata/AdminController.groovy b/grails-app/controllers/au/org/ala/ecodata/AdminController.groovy index 67a21b83a..7772d7d6c 100644 --- a/grails-app/controllers/au/org/ala/ecodata/AdminController.groovy +++ b/grails-app/controllers/au/org/ala/ecodata/AdminController.groovy @@ -272,7 +272,8 @@ class AdminController { dateFormat.setTimeZone(timeZoneUTC) def isMERIT = params.getBoolean('isMERIT', true) Date startDate = params.getDate("startDate", ["yyyy", "yyyy-MM-dd"]) ?: dateFormat.parse(defaultStartDate) - List siteIds = params.get("siteId")?.split(",") + List siteIds = params.get("siteId")?.split(",") ?: [] + List projectIds = params.get("projectId")?.split(",") ?: [] def code = 'success' def total = 0 def offset = 0 @@ -281,8 +282,14 @@ class AdminController { List defaultFids = metadataService.getSpatialLayerIdsToIntersect() log.debug("Number of fids to intersect: ${defaultFids.size()}; they are - ${defaultFids}") def totalSites - List projectIds = [] - if (siteIds) { + if (projectIds) { + projectIds.each { + siteIds.addAll(siteService.findAllSiteIdsForProject(it)) + } + + totalSites = siteIds.size() + } + else if (siteIds) { totalSites = siteIds.size() } else if (isMERIT) { diff --git a/grails-app/services/au/org/ala/ecodata/SpatialService.groovy b/grails-app/services/au/org/ala/ecodata/SpatialService.groovy index 82e3391a1..e48b1af84 100644 --- a/grails-app/services/au/org/ala/ecodata/SpatialService.groovy +++ b/grails-app/services/au/org/ala/ecodata/SpatialService.groovy @@ -4,10 +4,7 @@ import grails.core.GrailsApplication import grails.plugin.cache.Cacheable import groovy.json.JsonParserType import groovy.json.JsonSlurper -import org.locationtech.jts.geom.Coordinate -import org.locationtech.jts.geom.Geometry -import org.locationtech.jts.geom.GeometryCollection -import org.locationtech.jts.geom.GeometryFactory +import org.locationtech.jts.geom.* import org.locationtech.jts.io.WKTReader import static ParatooService.deepCopy @@ -73,8 +70,17 @@ class SpatialService { correctGeometry = geometryCollection.buffer(0) } - Geometry convexHullGeometry = correctGeometry.union().convexHull() - wkt = convexHullGeometry.toText() + Geometry convexHullGeometry + try { + convexHullGeometry = correctGeometry.union().convexHull() + } + catch (TopologyException e) { + log.error("Error creating convex hull for geometry collection") + convexHullGeometry = correctGeometry.buffer(0).union().convexHull() + } + finally { + wkt = convexHullGeometry?.toText() + } } String url = grailsApplication.config.getProperty('spatial.baseUrl')+WKT_INTERSECT_URL_PREFIX From ada424b71bf4b8e18c3afbb064581286040f5ead Mon Sep 17 00:00:00 2001 From: temi Date: Fri, 27 Sep 2024 12:54:43 +1000 Subject: [PATCH 04/19] AtlasOfLivingAustralia/fieldcapture#3292 - added force parameter to skip if calculation already done - fixed an issue with management unit site being skipped - get geo config of hub using project or current hub --- .../au/org/ala/ecodata/AdminController.groovy | 18 +++++++++++++----- .../au/org/ala/ecodata/MetadataService.groovy | 2 +- .../au/org/ala/ecodata/ProjectService.groovy | 16 ++++++++++++++++ .../au/org/ala/ecodata/SiteService.groovy | 16 +++++++++++++++- .../au/org/ala/ecodata/SiteServiceSpec.groovy | 4 ++-- 5 files changed, 47 insertions(+), 9 deletions(-) diff --git a/grails-app/controllers/au/org/ala/ecodata/AdminController.groovy b/grails-app/controllers/au/org/ala/ecodata/AdminController.groovy index 7772d7d6c..c61133147 100644 --- a/grails-app/controllers/au/org/ala/ecodata/AdminController.groovy +++ b/grails-app/controllers/au/org/ala/ecodata/AdminController.groovy @@ -270,7 +270,8 @@ class AdminController { def defaultStartDate = "2018-01-01" def timeZoneUTC = TimeZone.getTimeZone("UTC") dateFormat.setTimeZone(timeZoneUTC) - def isMERIT = params.getBoolean('isMERIT', true) + Boolean isForceFetch = params.getBoolean('force', true) + Boolean isMERIT = params.getBoolean('isMERIT', true) Date startDate = params.getDate("startDate", ["yyyy", "yyyy-MM-dd"]) ?: dateFormat.parse(defaultStartDate) List siteIds = params.get("siteId")?.split(",") ?: [] List projectIds = params.get("projectId")?.split(",") ?: [] @@ -332,13 +333,20 @@ class AdminController { log.info("${total+1} or ${(total+1)*100/totalSites} % sites updated in db..") } - if (!site.projects || !site.extent) { - log.debug("Ignoring site ${site.siteId} due to no associated projects or no extent") + if (!site.extent) { + log.debug("Ignoring site ${site.siteId} due to no extent") return } - def projectsOfSite = site.projects - List hubIds = projectService.findHubIdOfProjects(projectsOfSite) + // management unit site does not have any projects + def projectsOfSite = site.projects ?: [] + List hubIds = projectService.findHubIdFromProjectsOrCurrentHub(projectsOfSite) def fids = hubIds.size() == 1 ? metadataService.getSpatialLayerIdsToIntersect(hubIds[0]) : defaultFids + + if (!isForceFetch && siteService.areIntersectionCalculatedForAllLayers(site)) { + log.debug("Skipping site ${site.siteId} as all layers are already calculated and force fetch is not enabled - $isForceFetch") + return // Skip if all layers are already calculated + } + siteService.populateLocationMetadataForSite(site, fids) endInterimTime = System.currentTimeMillis() log.debug("Time taken to update metadata ${site.siteId}: ${endInterimTime - startInterimTime} ms") diff --git a/grails-app/services/au/org/ala/ecodata/MetadataService.groovy b/grails-app/services/au/org/ala/ecodata/MetadataService.groovy index 02f9885cb..eb95b00ef 100644 --- a/grails-app/services/au/org/ala/ecodata/MetadataService.groovy +++ b/grails-app/services/au/org/ala/ecodata/MetadataService.groovy @@ -490,7 +490,7 @@ class MetadataService { * @return */ List getSpatialLayerIdsToIntersectForProjects(List projectIds = []) { - List hubIds = projectService.findHubIdOfProjects(projectIds) + List hubIds = projectService.findHubIdFromProjectsOrCurrentHub(projectIds) hubIds.size() == 1 ? getSpatialLayerIdsToIntersect(hubIds[0]) : getSpatialLayerIdsToIntersect() } diff --git a/grails-app/services/au/org/ala/ecodata/ProjectService.groovy b/grails-app/services/au/org/ala/ecodata/ProjectService.groovy index 71afced38..e1ca2eaaa 100644 --- a/grails-app/services/au/org/ala/ecodata/ProjectService.groovy +++ b/grails-app/services/au/org/ala/ecodata/ProjectService.groovy @@ -51,6 +51,7 @@ class ProjectService { ActivityFormService activityFormService RecordService recordService LockService lockService + HubService hubService /* def getCommonService() { grailsApplication.mainContext.commonService @@ -1227,4 +1228,19 @@ class ProjectService { } } + /** + * Find hubs from project or use hubId query parameter + * @param projects + * @return + */ + def findHubIdFromProjectsOrCurrentHub (List projects) { + if (projects) { + return findHubIdOfProjects(projects) + } + else { + def currentHub = hubService.getCurrentHub() + return currentHub ? [currentHub.hubId] : [] + } + } + } \ No newline at end of file diff --git a/grails-app/services/au/org/ala/ecodata/SiteService.groovy b/grails-app/services/au/org/ala/ecodata/SiteService.groovy index 18e973980..a2db58243 100644 --- a/grails-app/services/au/org/ala/ecodata/SiteService.groovy +++ b/grails-app/services/au/org/ala/ecodata/SiteService.groovy @@ -708,7 +708,7 @@ class SiteService { */ def mergeIntersectionsArea(Map site, Map intersectionsAreaByFacets) { Map geometry = site.extent.geometry - List hubs = projectService.findHubIdOfProjects(site.projects) + List hubs = projectService.findHubIdFromProjectsOrCurrentHub(site.projects) String hubId = hubs?.size() == 1 ? hubs[0] : null Map existingIntersectionsArea = geometry[SpatialService.INTERSECTION_AREA] = geometry[SpatialService.INTERSECTION_AREA] ?: [:] intersectionsAreaByFacets?.each { String layer, Map nameAndValue -> @@ -1062,4 +1062,18 @@ class SiteService { List filterSitesByPurposeIsPlanning (List sites) { sites?.findAll { getPurpose(it) == Site.PLANNING_SITE_CODE } } + + /** + * Check if the intersection area is calculated for all provided layers + * @param site + * @param fids + * @return + */ + Boolean areIntersectionCalculatedForAllLayers(def site){ + List fids = metadataService.getGeographicConfig().checkForBoundaryIntersectionInLayers + fids?.every { fid -> + String group = metadataService.getGeographicFacetConfig(fid, null)?.name + site.extent?.geometry?[SpatialService.INTERSECTION_AREA]?[group]?[fid] != null + } + } } diff --git a/src/test/groovy/au/org/ala/ecodata/SiteServiceSpec.groovy b/src/test/groovy/au/org/ala/ecodata/SiteServiceSpec.groovy index bb215586a..fec77c79a 100644 --- a/src/test/groovy/au/org/ala/ecodata/SiteServiceSpec.groovy +++ b/src/test/groovy/au/org/ala/ecodata/SiteServiceSpec.groovy @@ -194,7 +194,7 @@ class SiteServiceSpec extends MongoSpec implements ServiceUnitTest def "New sites without a centroid should have one assigned"() { when: def result - projectService.findHubIdOfProjects(_) >> [] + projectService.findHubIdFromProjectsOrCurrentHub(_) >> [] Site.withSession { session -> result = service.create([name: 'Site 1', extent: [source: 'pid', geometry: [type: 'pid', pid: 'cl123']]]) session.flush() @@ -310,7 +310,7 @@ class SiteServiceSpec extends MongoSpec implements ServiceUnitTest def "The site area is calculated from the FeatureCollection for a compound site"() { setup: - projectService.findHubIdOfProjects(_) >> [] + projectService.findHubIdFromProjectsOrCurrentHub(_) >> [] def coordinates = [[148.260498046875, -37.26530995561874], [148.260498046875, -37.26531995561874], [148.310693359375, -37.26531995561874], [148.310693359375, -37.26531995561874], [148.260498046875, -37.26530995561874]] def extent = buildExtent('drawn', 'Polygon', coordinates) Map site = [type: Site.TYPE_COMPOUND, extent: extent, features: [ From 89fe4a098e12699e3e4abf382eb7a2d0d0df091f Mon Sep 17 00:00:00 2001 From: Jack Brinkman Date: Fri, 27 Sep 2024 13:36:34 +1000 Subject: [PATCH 05/19] Removed threshhold & tolerance log from getSimpleProjectArea, added try-catch --- .../au/org/ala/ecodata/SiteService.groovy | 23 +++++++++++-------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/grails-app/services/au/org/ala/ecodata/SiteService.groovy b/grails-app/services/au/org/ala/ecodata/SiteService.groovy index 3bb3e48f7..06cc655fe 100644 --- a/grails-app/services/au/org/ala/ecodata/SiteService.groovy +++ b/grails-app/services/au/org/ala/ecodata/SiteService.groovy @@ -258,21 +258,24 @@ class SiteService { def getSimpleProjectArea(projectSiteId) { def threshold = grailsApplication.config.getProperty('biocollect.projectArea.simplificationThreshold', Integer, 10000) def tolerance = grailsApplication.config.getProperty('biocollect.projectArea.simplificationTolerance', Double, 0.0001) - log.info("Threshhold ${threshold} Tolerance ${tolerance}") def site = get(projectSiteId, [SiteService.FLAT, SiteService.INDEXING]) - if (site != null) { - def projectArea = geometryAsGeoJson(site) - - if (projectArea?.coordinates != null) { - def coordsSize = projectArea.coordinates.flatten().size() - if (coordsSize > threshold) { - site.geoIndex = GeometryUtils.simplify(projectArea, tolerance) - } else { - site.geoIndex = projectArea + try { + if (site != null) { + def projectArea = geometryAsGeoJson(site) + + if (projectArea?.coordinates != null) { + def coordsSize = projectArea.coordinates.flatten().size() + if (coordsSize > threshold) { + site.geoIndex = GeometryUtils.simplify(projectArea, tolerance) + } else { + site.geoIndex = projectArea + } } } + } catch (Exception e) { + log.info("Unable to get simplified project area geometry (site ${site.siteId}") } site From d3a55c84f1bd9aae49cdf0fa6572d8b8ff95766d Mon Sep 17 00:00:00 2001 From: temi Date: Fri, 27 Sep 2024 15:02:21 +1000 Subject: [PATCH 06/19] AtlasOfLivingAustralia/fieldcapture#3292 - added logging statement --- .../controllers/au/org/ala/ecodata/AdminController.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/grails-app/controllers/au/org/ala/ecodata/AdminController.groovy b/grails-app/controllers/au/org/ala/ecodata/AdminController.groovy index c61133147..5432ccbdd 100644 --- a/grails-app/controllers/au/org/ala/ecodata/AdminController.groovy +++ b/grails-app/controllers/au/org/ala/ecodata/AdminController.groovy @@ -360,7 +360,7 @@ class AdminController { } } catch (Exception e) { - log.error("Unable to complete the operation ", e) + log.error("Unable to complete the operation for siteId - ${site.siteId} ", e) code = "error" } } From fc1094d73f36d7799ca1c9e762b8f0497d54e32a Mon Sep 17 00:00:00 2001 From: temi Date: Fri, 27 Sep 2024 16:16:52 +1000 Subject: [PATCH 07/19] AtlasOfLivingAustralia/fieldcapture#3292 - adding manual assinged electorates / states to download --- .../ala/ecodata/reporting/ProjectXlsExporter.groovy | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/main/groovy/au/org/ala/ecodata/reporting/ProjectXlsExporter.groovy b/src/main/groovy/au/org/ala/ecodata/reporting/ProjectXlsExporter.groovy index 2d535c64c..0887abb8e 100644 --- a/src/main/groovy/au/org/ala/ecodata/reporting/ProjectXlsExporter.groovy +++ b/src/main/groovy/au/org/ala/ecodata/reporting/ProjectXlsExporter.groovy @@ -339,6 +339,19 @@ class ProjectXlsExporter extends ProjectExporter { else log.error ("No facet config found for layer $layer.") } + + if (project.geographicInfo) { + // load from manually assigned electorates/states + if (!project.containsKey(getPropertyNameForFacet("elect"))) { + project[getPropertyNameForFacet("elect")] = project.geographicInfo.primaryElectorate + project[getPropertyNameForFacet("elect","other")] = project.geographicInfo.otherElectorates?.join("; ") + } + + if (!project.containsKey(getPropertyNameForFacet("state"))) { + project[getPropertyNameForFacet("state")] = project.geographicInfo.primaryState + project[getPropertyNameForFacet("state","other")] = project.geographicInfo.otherStates?.join("; ") + } + } } private addProjectGeo(Map project) { From f30e99ddbfc3d8edb052dcad4ab55e2510978cee Mon Sep 17 00:00:00 2001 From: Jack Brinkman Date: Mon, 30 Sep 2024 15:30:59 +1000 Subject: [PATCH 08/19] SciStarter import job is now configurable --- grails-app/jobs/au/org/ala/ecodata/ImportSciStarterJob.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/grails-app/jobs/au/org/ala/ecodata/ImportSciStarterJob.groovy b/grails-app/jobs/au/org/ala/ecodata/ImportSciStarterJob.groovy index eec02b3c1..b861010cd 100644 --- a/grails-app/jobs/au/org/ala/ecodata/ImportSciStarterJob.groovy +++ b/grails-app/jobs/au/org/ala/ecodata/ImportSciStarterJob.groovy @@ -10,7 +10,7 @@ class ImportSciStarterJob { static triggers = { Boolean enabled = grailsApplication.config.getProperty("sciStarter.importEnabled", Boolean, true) if (enabled) { - cron name: "every sunday", cronExpression: "0 0 0 ? * 1/7 *" + cron name: "every sunday", cronExpression: grailsApplication.config.getProperty("scistarter.cronSchedule", "0 0 0 ? * 1/7 *") } } From c3e98cc40576b7e312980d1a18960d4a6d470778 Mon Sep 17 00:00:00 2001 From: Jack Brinkman Date: Tue, 1 Oct 2024 17:14:03 +1000 Subject: [PATCH 09/19] Removed worldExtent project site default for sciStarter projects without a project area --- grails-app/conf/data/worldExtent.json | 97 ------------------- .../au/org/ala/ecodata/ProjectService.groovy | 20 +--- 2 files changed, 1 insertion(+), 116 deletions(-) delete mode 100644 grails-app/conf/data/worldExtent.json diff --git a/grails-app/conf/data/worldExtent.json b/grails-app/conf/data/worldExtent.json deleted file mode 100644 index 126a98969..000000000 --- a/grails-app/conf/data/worldExtent.json +++ /dev/null @@ -1,97 +0,0 @@ -{ - "projects" : [ - ], - "isSciStarter":true, - "status" : "active", - "poi" : [ ], - "geoIndex" : { - "type" : "Polygon", - "coordinates" : [ - [ - [ - -180, - -90 - ], - [ - -180, - 90 - ], - [ - 180, - 90 - ], - [ - 180, - -90 - ], - [ - -180, - -90 - ] - ] - ] - }, - "extent" : { - "source" : "drawn", - "geometry" : { - "fid" : "", - "mvs" : null, - "precision" : "", - "centre" : [ - "11.601562499999986", - "-0.7859697100237464" - ], - "state" : null, - "layerName" : "", - "pid" : null, - "datum" : "", - "type" : "Polygon", - "decimalLatitude" : -0.7859697100237497, - "areaKmSq" : 509700727.0660377, - "nrm" : null, - "bbox" : "", - "mvg" : null, - "name" : "", - "decimalLongitude" : 11.601562499999986, - "locality" : "Lolo-Bouenguidi, Gabon", - "lga" : null, - "uncertainty" : "", - "coordinates" : [ - [ - [ - -180, - -90 - ], - [ - -180, - 90 - ], - [ - 180, - 90 - ], - [ - 180, - -90 - ], - [ - -180, - -90 - ] - ] - ], - "ibra" : null, - "imcra4_pb" : null, - "elect" : null, - "cmz" : null, - "other" : [ ], - "gerSubRegion" : [ ] - } - }, - "externalId" : "", - "area" : "5.097007270660377E8", - "description" : "Default map for SciStarter projects without a predefined region.", - "name" : "SciStarter Project Area", - "notes" : "", - "type" : "projectArea" -} \ No newline at end of file diff --git a/grails-app/services/au/org/ala/ecodata/ProjectService.groovy b/grails-app/services/au/org/ala/ecodata/ProjectService.groovy index 47af6127f..3ebe19209 100644 --- a/grails-app/services/au/org/ala/ecodata/ProjectService.groovy +++ b/grails-app/services/au/org/ala/ecodata/ProjectService.groovy @@ -881,7 +881,7 @@ class ProjectService { * @return */ Map createSciStarterSites(Map project) { - Map result = [siteIds: null] + Map result = [siteIds: []] List sites = [] if (project.regions) { // convert region to site @@ -895,12 +895,6 @@ class ProjectService { } result.siteIds = sites - } else { - // if no region, then create world extent. - String siteId = getWorldExtent() - if (siteId) { - result.siteIds = [siteId] - } } result @@ -977,18 +971,6 @@ class ProjectService { } } - /** - * World extent is used as project area of all SciStarter Projects without project area. This function - * creates a new world extent every time it is called. - * @return - siteId - 'abcd-sds' - */ - String getWorldExtent() { - // use JSON.parse since JSONSlurper converts numbers to BigDecimal which throws error on serialization. - Object world = JSON.parse(getClass().getResourceAsStream("/data/worldExtent.json")?.getText()) - Map site = siteService.create(world) - return site.siteId - } - /** * Get a String that describes a project. This used for indexing purposes currently. * The motivaion for this method was because there was no single field to distinguish between projects but had multiple From 80d510a81b514005c976a87675a865661c601f4e Mon Sep 17 00:00:00 2001 From: Jack Brinkman Date: Wed, 2 Oct 2024 10:07:17 +1000 Subject: [PATCH 10/19] Temporarily disabled SciStarter site indexing --- .../au/org/ala/ecodata/ProjectService.groovy | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/grails-app/services/au/org/ala/ecodata/ProjectService.groovy b/grails-app/services/au/org/ala/ecodata/ProjectService.groovy index 3ebe19209..24b262983 100644 --- a/grails-app/services/au/org/ala/ecodata/ProjectService.groovy +++ b/grails-app/services/au/org/ala/ecodata/ProjectService.groovy @@ -814,13 +814,13 @@ class ProjectService { Map organisation // create project extent - Map sites = createSciStarterSites(rawProp) - String projectSiteId - if (sites?.siteIds?.size()) { - projectSiteId = sites.siteIds[0] - } - - transformedProp.projectSiteId = projectSiteId +// Map sites = createSciStarterSites(rawProp) +// String projectSiteId +// if (sites?.siteIds?.size()) { +// projectSiteId = sites.siteIds[0] +// } +// +// transformedProp.projectSiteId = projectSiteId // create organisation if (transformedProp.organisationName) { @@ -842,9 +842,9 @@ class ProjectService { // use the projectId to associate site with project if (projectId) { - sites?.siteIds?.each { siteId -> - siteService.addProject(siteId, projectId) - } +// sites?.siteIds?.each { siteId -> +// siteService.addProject(siteId, projectId) +// } // create project logo. createSciStarterLogo(imageUrl, attribution, projectId) From 20cc0efe1a3b6bddbe84e78c97b6d0b6ac257710 Mon Sep 17 00:00:00 2001 From: Jack Brinkman Date: Wed, 2 Oct 2024 13:42:59 +1000 Subject: [PATCH 11/19] Revert "Temporarily disabled SciStarter site indexing" This reverts commit 80d510a81b514005c976a87675a865661c601f4e. --- .../au/org/ala/ecodata/ProjectService.groovy | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/grails-app/services/au/org/ala/ecodata/ProjectService.groovy b/grails-app/services/au/org/ala/ecodata/ProjectService.groovy index 24b262983..3ebe19209 100644 --- a/grails-app/services/au/org/ala/ecodata/ProjectService.groovy +++ b/grails-app/services/au/org/ala/ecodata/ProjectService.groovy @@ -814,13 +814,13 @@ class ProjectService { Map organisation // create project extent -// Map sites = createSciStarterSites(rawProp) -// String projectSiteId -// if (sites?.siteIds?.size()) { -// projectSiteId = sites.siteIds[0] -// } -// -// transformedProp.projectSiteId = projectSiteId + Map sites = createSciStarterSites(rawProp) + String projectSiteId + if (sites?.siteIds?.size()) { + projectSiteId = sites.siteIds[0] + } + + transformedProp.projectSiteId = projectSiteId // create organisation if (transformedProp.organisationName) { @@ -842,9 +842,9 @@ class ProjectService { // use the projectId to associate site with project if (projectId) { -// sites?.siteIds?.each { siteId -> -// siteService.addProject(siteId, projectId) -// } + sites?.siteIds?.each { siteId -> + siteService.addProject(siteId, projectId) + } // create project logo. createSciStarterLogo(imageUrl, attribution, projectId) From 9ccce4064c0cb643f46735963ff9b7eba4c09489 Mon Sep 17 00:00:00 2001 From: temi Date: Wed, 2 Oct 2024 14:12:37 +1000 Subject: [PATCH 12/19] Revert "Removed worldExtent project site default for sciStarter projects without a project area" This reverts commit c3e98cc40576b7e312980d1a18960d4a6d470778. --- grails-app/conf/data/worldExtent.json | 97 +++++++++++++++++++ .../au/org/ala/ecodata/ProjectService.groovy | 20 +++- 2 files changed, 116 insertions(+), 1 deletion(-) create mode 100644 grails-app/conf/data/worldExtent.json diff --git a/grails-app/conf/data/worldExtent.json b/grails-app/conf/data/worldExtent.json new file mode 100644 index 000000000..126a98969 --- /dev/null +++ b/grails-app/conf/data/worldExtent.json @@ -0,0 +1,97 @@ +{ + "projects" : [ + ], + "isSciStarter":true, + "status" : "active", + "poi" : [ ], + "geoIndex" : { + "type" : "Polygon", + "coordinates" : [ + [ + [ + -180, + -90 + ], + [ + -180, + 90 + ], + [ + 180, + 90 + ], + [ + 180, + -90 + ], + [ + -180, + -90 + ] + ] + ] + }, + "extent" : { + "source" : "drawn", + "geometry" : { + "fid" : "", + "mvs" : null, + "precision" : "", + "centre" : [ + "11.601562499999986", + "-0.7859697100237464" + ], + "state" : null, + "layerName" : "", + "pid" : null, + "datum" : "", + "type" : "Polygon", + "decimalLatitude" : -0.7859697100237497, + "areaKmSq" : 509700727.0660377, + "nrm" : null, + "bbox" : "", + "mvg" : null, + "name" : "", + "decimalLongitude" : 11.601562499999986, + "locality" : "Lolo-Bouenguidi, Gabon", + "lga" : null, + "uncertainty" : "", + "coordinates" : [ + [ + [ + -180, + -90 + ], + [ + -180, + 90 + ], + [ + 180, + 90 + ], + [ + 180, + -90 + ], + [ + -180, + -90 + ] + ] + ], + "ibra" : null, + "imcra4_pb" : null, + "elect" : null, + "cmz" : null, + "other" : [ ], + "gerSubRegion" : [ ] + } + }, + "externalId" : "", + "area" : "5.097007270660377E8", + "description" : "Default map for SciStarter projects without a predefined region.", + "name" : "SciStarter Project Area", + "notes" : "", + "type" : "projectArea" +} \ No newline at end of file diff --git a/grails-app/services/au/org/ala/ecodata/ProjectService.groovy b/grails-app/services/au/org/ala/ecodata/ProjectService.groovy index 3ebe19209..47af6127f 100644 --- a/grails-app/services/au/org/ala/ecodata/ProjectService.groovy +++ b/grails-app/services/au/org/ala/ecodata/ProjectService.groovy @@ -881,7 +881,7 @@ class ProjectService { * @return */ Map createSciStarterSites(Map project) { - Map result = [siteIds: []] + Map result = [siteIds: null] List sites = [] if (project.regions) { // convert region to site @@ -895,6 +895,12 @@ class ProjectService { } result.siteIds = sites + } else { + // if no region, then create world extent. + String siteId = getWorldExtent() + if (siteId) { + result.siteIds = [siteId] + } } result @@ -971,6 +977,18 @@ class ProjectService { } } + /** + * World extent is used as project area of all SciStarter Projects without project area. This function + * creates a new world extent every time it is called. + * @return - siteId - 'abcd-sds' + */ + String getWorldExtent() { + // use JSON.parse since JSONSlurper converts numbers to BigDecimal which throws error on serialization. + Object world = JSON.parse(getClass().getResourceAsStream("/data/worldExtent.json")?.getText()) + Map site = siteService.create(world) + return site.siteId + } + /** * Get a String that describes a project. This used for indexing purposes currently. * The motivaion for this method was because there was no single field to distinguish between projects but had multiple From a0a9f340645e3b8a64386fc9a5c703aa169df8a4 Mon Sep 17 00:00:00 2001 From: temi Date: Wed, 2 Oct 2024 14:14:23 +1000 Subject: [PATCH 13/19] fix for total field index limit on ES --- grails-app/services/au/org/ala/ecodata/SiteService.groovy | 2 ++ 1 file changed, 2 insertions(+) diff --git a/grails-app/services/au/org/ala/ecodata/SiteService.groovy b/grails-app/services/au/org/ala/ecodata/SiteService.groovy index 06cc655fe..ee25d59ed 100644 --- a/grails-app/services/au/org/ala/ecodata/SiteService.groovy +++ b/grails-app/services/au/org/ala/ecodata/SiteService.groovy @@ -278,6 +278,8 @@ class SiteService { log.info("Unable to get simplified project area geometry (site ${site.siteId}") } + // remove extent to not avoid total fields limit in ES + site?.remove('extent') site } From e212fbf8d9e15e4c22074de74436af4503eaa14a Mon Sep 17 00:00:00 2001 From: Jack Brinkman Date: Wed, 2 Oct 2024 17:14:30 +1000 Subject: [PATCH 14/19] Improved logging of getSimpleProjectArea errors --- grails-app/services/au/org/ala/ecodata/SiteService.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/grails-app/services/au/org/ala/ecodata/SiteService.groovy b/grails-app/services/au/org/ala/ecodata/SiteService.groovy index ee25d59ed..7e656c58c 100644 --- a/grails-app/services/au/org/ala/ecodata/SiteService.groovy +++ b/grails-app/services/au/org/ala/ecodata/SiteService.groovy @@ -275,7 +275,7 @@ class SiteService { } } } catch (Exception e) { - log.info("Unable to get simplified project area geometry (site ${site.siteId}") + log.info("Unable to get simplified project area geometry (site ${site.siteId})", e) } // remove extent to not avoid total fields limit in ES From 31a60695e3370283df6429a06bebbdc263d46b38 Mon Sep 17 00:00:00 2001 From: temi Date: Thu, 3 Oct 2024 07:05:06 +1000 Subject: [PATCH 15/19] fix for long string --- src/main/groovy/au/org/ala/ecodata/GeometryUtils.groovy | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/groovy/au/org/ala/ecodata/GeometryUtils.groovy b/src/main/groovy/au/org/ala/ecodata/GeometryUtils.groovy index 0d73edeee..c194aea87 100644 --- a/src/main/groovy/au/org/ala/ecodata/GeometryUtils.groovy +++ b/src/main/groovy/au/org/ala/ecodata/GeometryUtils.groovy @@ -29,7 +29,8 @@ class GeometryUtils { static Map wktToGeoJson(String wkt, int decimals = 20) { WKTReader wktReader = new WKTReader() - Geometry geom = wktReader.read(wkt) + def reader = new StringReader(wkt) + Geometry geom = wktReader.read(reader) String geoJSON = new GeometryJSON(decimals).toString(geom) ObjectMapper mapper = new ObjectMapper() return mapper.readValue(geoJSON, Map) From ab4abdeefb9b4683d76cb323d9cc5d859b9be6bc Mon Sep 17 00:00:00 2001 From: temi Date: Thu, 3 Oct 2024 09:44:24 +1000 Subject: [PATCH 16/19] fix for toString exception --- src/main/groovy/au/org/ala/ecodata/GeometryUtils.groovy | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/main/groovy/au/org/ala/ecodata/GeometryUtils.groovy b/src/main/groovy/au/org/ala/ecodata/GeometryUtils.groovy index c194aea87..bda514e80 100644 --- a/src/main/groovy/au/org/ala/ecodata/GeometryUtils.groovy +++ b/src/main/groovy/au/org/ala/ecodata/GeometryUtils.groovy @@ -18,6 +18,8 @@ import org.opengis.referencing.crs.CoordinateReferenceSystem import org.opengis.referencing.operation.MathTransform import java.awt.geom.Point2D +import java.nio.charset.StandardCharsets + /** * Helper class for working with site geometry. */ @@ -29,11 +31,13 @@ class GeometryUtils { static Map wktToGeoJson(String wkt, int decimals = 20) { WKTReader wktReader = new WKTReader() + ByteArrayOutputStream outputStream = new ByteArrayOutputStream() + OutputStreamWriter writer = new OutputStreamWriter(outputStream, StandardCharsets.UTF_8) def reader = new StringReader(wkt) Geometry geom = wktReader.read(reader) - String geoJSON = new GeometryJSON(decimals).toString(geom) + new GeometryJSON(decimals).write(geom, writer) ObjectMapper mapper = new ObjectMapper() - return mapper.readValue(geoJSON, Map) + return mapper.readValue(outputStream.toByteArray(), Map) } static String wktToMultiPolygonWkt(String wkt) { From a3f7d9ddfb02fd7b4cba6bb9102b2163745811cf Mon Sep 17 00:00:00 2001 From: temi Date: Thu, 3 Oct 2024 14:38:18 +1000 Subject: [PATCH 17/19] release 5.0 --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 09bbe4ed1..6ad6af72c 100644 --- a/build.gradle +++ b/build.gradle @@ -20,7 +20,7 @@ plugins { id "com.gorylenko.gradle-git-properties" version "2.4.1" } -version "5.0-SNAPSHOT" +version "5.0" group "au.org.ala" description "Ecodata" From 171991426cec23904b4050ed9501455d01ef6c02 Mon Sep 17 00:00:00 2001 From: temi Date: Thu, 3 Oct 2024 17:04:56 +1000 Subject: [PATCH 18/19] release 5.0.1 --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 6ad6af72c..51515114d 100644 --- a/build.gradle +++ b/build.gradle @@ -20,7 +20,7 @@ plugins { id "com.gorylenko.gradle-git-properties" version "2.4.1" } -version "5.0" +version "5.0.1" group "au.org.ala" description "Ecodata" From 6d9552adcd4307409f35a6c4fc9bf0ed3513453e Mon Sep 17 00:00:00 2001 From: temi Date: Thu, 3 Oct 2024 17:10:42 +1000 Subject: [PATCH 19/19] 5.1-SNAPSHOT --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 51515114d..2c18d5340 100644 --- a/build.gradle +++ b/build.gradle @@ -20,7 +20,7 @@ plugins { id "com.gorylenko.gradle-git-properties" version "2.4.1" } -version "5.0.1" +version "5.1-SNAPSHOT" group "au.org.ala" description "Ecodata"