Merge pull request #895 from AtlasOfLivingAustralia/feature/paratoo

Feature/paratoo

grails-app/domain/au/org/ala/ecodata/ExternalId.groovy

@@ -12,7 +12,7 @@ class ExternalId implements Comparable {
 
     enum IdType {
         INTERNAL_ORDER_NUMBER, TECH_ONE_CODE, WORK_ORDER, GRANT_AWARD, GRANT_OPPORTUNITY, RELATED_PROJECT,
-        MONITOR_PROTOCOL_INTERNAL_ID, MONITOR_PROTOCOL_GUID, TECH_ONE_CONTRACT_NUMBER }
+        MONITOR_PROTOCOL_INTERNAL_ID, MONITOR_PROTOCOL_GUID, TECH_ONE_CONTRACT_NUMBER, MONITOR_PLOT_GUID, UNSPECIFIED }
 
     static constraints = {
     }

grails-app/domain/au/org/ala/ecodata/Site.groovy

@@ -25,18 +25,22 @@ class Site {
         sites may have 0..n Activities/Assessments - mapped from the Activity side
     */
 
+    static hasMany = [externalIds:ExternalId]
+    static embedded = ['externalIds']
+
     static mapping = {
         name index: true
         siteId index: true
         projects index: true
         version false
     }
 
+    static transients = ['externalId', 'defaultExternalId']
     ObjectId id
     String siteId
     String status = 'active'
     String visibility
-    String externalId
+
     List projects = []
     String name
     String type
@@ -92,6 +96,7 @@ class Site {
                 }
             }
         }
+        externalIds nullable: true
     }
 
     def getAssociations(){
@@ -131,4 +136,49 @@ class Site {
             }
         }
     }
+
+    /**
+     * The MERIT and BioCollect UIs have a site page that allows the user to enter an external Id
+     * for the Site without specifying the type.
+     * Hence we treat this as the "Default" external Id which is either the first unspecified external Id
+     * or the first external Id if there are no unspecified external Ids.
+     * @return
+     */
+    private ExternalId defaultExternalId() {
+        if (!externalIds) {
+            return null
+        }
+        ExternalId defaultExternalId = externalIds?.find {it.idType == ExternalId.IdType.UNSPECIFIED}
+        if (!defaultExternalId) {
+            defaultExternalId = externalIds[0]
+        }
+        defaultExternalId
+    }
+
+    String getExternalId() {
+        ExternalId defaultExternalId = defaultExternalId()
+        defaultExternalId?.externalId
+    }
+
+    void setExternalId(String externalId) {
+        ExternalId defaultExternalId = defaultExternalId()
+        if (defaultExternalId) {
+            defaultExternalId.externalId = externalId
+
+        } else {
+            externalIds = [new ExternalId(idType: ExternalId.IdType.UNSPECIFIED, externalId: externalId)]
+        }
+        markDirty('externalIds')
+    }
+
+
+    static Site findByExternalId(ExternalId.IdType idType, String externalId) {
+        where {
+            externalIds {
+                idType == idType
+                externalId == externalId
+            }
+            status != Status.DELETED
+        }.find()
+    }
 }

grails-app/services/au/org/ala/ecodata/ParatooService.groovy

@@ -233,7 +233,10 @@ class ParatooService {
             siteProps.type = Site.TYPE_SURVEY_AREA
             siteProps.publicationStatus = PublicationStatus.PUBLISHED
             siteProps.projects = [project.projectId]
-            Site site = Site.findByTypeAndExternalId(Site.TYPE_SURVEY_AREA, siteProps.externalId)
+            if (geoJson.properties?.externalId) {
+                siteProps.externalIds = [new ExternalId(idType:ExternalId.IdType.MONITOR_PLOT_GUID, externalId: geoJson.properties.externalId)]
+            }
+            Site site = Site.findByExternalId(ExternalId.IdType.MONITOR_PLOT_GUID, siteProps.externalId)
             Map result
             if (!site) {
                 result = siteService.create(siteProps)
@@ -401,7 +404,7 @@ class ParatooService {
         // The project/s for the site will be specified by a subsequent call to /projects
         siteData.projects = []
 
-        Site site = Site.findByExternalId(siteData.externalId)
+        Site site = Site.findByExternalId(ExternalId.IdType.MONITOR_PLOT_GUID, siteData.externalId)
         Map result
         if (site) {
             result = siteService.update(siteData, site.siteId)
@@ -418,6 +421,7 @@ class ParatooService {
         Map site = SiteService.propertiesFromGeoJson(geoJson, 'point')
         site.projects = [] // get all projects for the user I suppose - not sure why this isn't in the payload as it's in the UI...
         site.type = Site.TYPE_SURVEY_AREA
+        site.externalIds = [new ExternalId(idType:ExternalId.IdType.MONITOR_PLOT_GUID, externalId:geoJson.properties.externalId)]
 
         site
     }
@@ -437,26 +441,32 @@ class ParatooService {
     }
 
 
-    private Map linkProjectToSites(ParatooProject project, List siteExternalIds, List<ParatooProject> userProjects) {
+    private static Map linkProjectToSites(ParatooProject project, List siteExternalIds, List<ParatooProject> userProjects) {
         List errors = []
 
-        List<Site> sites = Site.findAllByTypeAndExternalIdInList(Site.TYPE_SURVEY_AREA, siteExternalIds)
-        sites.each { Site site ->
-            site.projects = site.projects ?: []
-            if (!site.projects.contains(project.id)) {
-                // Validate that the user has permission to link the site to the project by checking
-                // if the user has permission on any other projects this site is linked to.
-                if (site.projects) {
-                    if (!userProjects.collect{it.id}.containsAll(site.projects)) {
-                        errors << "User does not have permission to link site ${site.externalId} to project ${project.id}"
-                        return
+        siteExternalIds.each { String siteExternalId ->
+
+            Site site = Site.findByExternalId(ExternalId.IdType.MONITOR_PLOT_GUID, siteExternalId)
+            if (site) {
+                site.projects = site.projects ?: []
+                if (!site.projects.contains(project.id)) {
+                    // Validate that the user has permission to link the site to the project by checking
+                    // if the user has permission on any other projects this site is linked to.
+                    if (site.projects) {
+                        if (!userProjects.collect{it.id}.containsAll(site.projects)) {
+                            errors << "User does not have permission to link site ${site.externalId} to project ${project.id}"
+                            return
+                        }
+                    }
+                    site.projects << project.id
+                    site.save()
+                    if (site.hasErrors()) {
+                        errors << site.errors
                     }
                 }
-                site.projects << project.id
-                site.save()
-                if (site.hasErrors()) {
-                    errors << site.errors
-                }
+            }
+            else {
+                errors << "No site exists with externalId = ${siteExternalId}"
             }
         }
         [success:!errors, error:errors]
@@ -484,20 +494,4 @@ class ParatooService {
             siteService.create(site)
         }
     }
-
-    // Protocol = 2 (vegetation mapping survey).
-        // endpoint /api/vegetation-mapping-surveys is useless
-        // (possibly because the protocol is called vegetation-mapping-surveys and there is a module/component inside
-        // called vegetation-mapping-survey and the strapi pluralisation is causing issues?)
-        // Instead if you query: https://dev.core-api.paratoo.tern.org.au/api/vegetation-mapping-observations?populate=deep
-        // You can get multiple observations with different points linked to the same surveyId.
-
-        // Protocol = 7 (drone survey) - No useful spatial data
-
-        // Protocol = 10 & 11 (Photopoints - Compact Panorama & Photopoints - Device Panorama) - same endpoint.
-        // Has plot-layout / plot-visit
-
-        // Protocol = 12 (Floristics - enhanced)
-        // Has plot-layout / plot-visit
-
 }

grails-app/services/au/org/ala/ecodata/PermissionService.groovy

@@ -224,7 +224,7 @@ class PermissionService {
             out.put(it.userId,rec);
 
         }
-        def userList = authService.getUserDetailsById(userIds)
+        def userList = userIds ? authService.getUserDetailsById(userIds) : null
 
         if (userList) {
             def users = userList['users']
@@ -343,7 +343,7 @@ class PermissionService {
             out.put(it.userId,toMap(it,false))
         }
 
-        def userList = authService.getUserDetailsById(userIds)
+        def userList = userIds ? authService.getUserDetailsById(userIds) : null
         if (userList) {
             def users = userList['users']
 

grails-app/services/au/org/ala/ecodata/SiteService.groovy

@@ -70,6 +70,9 @@ class SiteService {
             def map = [:]
             if (levelOfDetail.contains(RAW)) {
                 map = commonService.toBareMap(o)
+                // Treat the default externalId as the externalId property for backwards
+                // compatibility with the MERIT and BioCollect UI
+                map['externalId'] = o.externalId
             } else {
                 map = toMap(o, levelOfDetail)
             }
@@ -141,7 +144,11 @@ class SiteService {
      */
     def toMap(site, levelOfDetail = [], version = null) {
         def mapOfProperties = site instanceof Site ? GormMongoUtil.extractDboProperties(site.getProperty("dbo")) : site
-       // def mapOfProperties = site instanceof Site ? site.getProperty("dbo") : site
+        // Treat the default externalId as the externalId property for backwards
+        // compatibility with the MERIT and BioCollect UI
+        if (site instanceof Site) {
+            mapOfProperties['externalId'] = site.externalId
+        }
         def id = mapOfProperties["_id"].toString()
         mapOfProperties["id"] = id
         mapOfProperties.remove("_id")
@@ -212,7 +219,7 @@ class SiteService {
             properties = geoJson.properties
             geometry = geoJson.geometry
         }
-        Map site = [name:properties.name, description:properties.description, externalId:properties.externalId, notes:properties.notes]
+        Map site = [name:properties.name, description:properties.description, notes:properties.notes]
         site.extent = [geometry:geometry, source:source]
 
         if (geometry.type == 'Point') {
@@ -293,8 +300,12 @@ class SiteService {
             }
         }
 
-        //getCommonService().updateProperties(site, props)
+        // The commonService won't update externalId because it's a transient property
+        if (props.externalId) {
+            site.externalId = props.externalId
+        }
         commonService.updateProperties(site, props)
+
     }
 
 

scripts/releases/4.4/fixAndMigrateSiteExternalIds.js

@@ -0,0 +1,30 @@
+load('../../utils/audit.js');
+let adminUserId = 'system';
+let sites = db.site.find({externalId:{$type:'number'}});
+while (sites.hasNext()) {
+    let site = sites.next();
+    site.externalId = new String(site.externalId);
+    db.site.replaceOne({_id:site._id}, site);
+
+    audit(site, site.siteId, 'au.org.ala.ecodata.Site', adminUserId);
+}
+
+
+sites = db.site.find({externalId:{$type:'object'}});
+while (sites.hasNext()) {
+    let site = sites.next();
+    site.externalId = null;
+    db.site.replaceOne({_id:site._id}, site);
+
+    audit(site, site.siteId, 'au.org.ala.ecodata.Site', adminUserId);
+}
+
+sites = db.site.find({externalId:{$ne:null}});
+while (sites.hasNext()) {
+    let site = sites.next();
+    site.externalIds = [{idType:'UNSPECIFIED', externalId: site.externalId}];
+    delete site.externalId;
+    db.site.replaceOne({_id:site._id}, site);
+
+    audit(site, site.siteId, 'au.org.ala.ecodata.Site', adminUserId);
+}

src/main/groovy/au/org/ala/ecodata/IniReader.java

@@ -138,13 +138,13 @@ public String getStringValue(String section, String key) {
      */
     public int getIntegerValue(String section, String key) {
         String str = document.get(section + "\\" + key);
-        Integer ret;
+        int ret;
         try {
-            ret = new Integer(str);
+            ret = Integer.parseInt(str);
         } catch (Exception e) {
-            ret = new Integer(0);
+            ret = 0;
         }
-        return ret.intValue();
+        return ret;
     }
 
     /**

src/test/groovy/au/org/ala/ecodata/ParatooJsonViewSpec.groovy

@@ -78,6 +78,6 @@ class ParatooJsonViewSpec extends Specification implements JsonViewTest {
     }
 
     private Site buildSite(i) {
-        new Site(siteId:"s$i", externalId:"s$i", name:"Site $i", extent:[geometry:[type:'Polygon', coordinates:DUMMY_POLYGON]])
+        new Site(siteId:"s$i", externalIds:[[idType:ExternalId.IdType.MONITOR_PROTOCOL_GUID, externalId:"s$i"]], name:"Site $i", extent:[geometry:[type:'Polygon', coordinates:DUMMY_POLYGON]])
     }
 }

src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy

@@ -168,7 +168,7 @@ class ParatooServiceSpec extends MongoSpec implements ServiceUnitTest<ParatooSer
                 "uuid":"lmpisy5p9g896lad4ut",
                 "comment":"Test"]
 
-        Map expected = ['name':'CTMAUA2222', 'description':'CTMAUA2222', 'externalId':'lmpisy5p9g896lad4ut', 'notes':'Test', 'extent':['geometry':['type':'Point', 'coordinates':[149.0651439, -35.2592424], 'decimalLatitude':-35.2592424, 'decimalLongitude':149.0651439], 'source':'point'], 'projects':[], 'type':'surveyArea']
+        Map expected = ['name':'CTMAUA2222', 'description':'CTMAUA2222', 'externalIds':[new ExternalId(externalId:'lmpisy5p9g896lad4ut', idType:ExternalId.IdType.MONITOR_PLOT_GUID)], 'notes':'Test', 'extent':['geometry':['type':'Point', 'coordinates':[149.0651439, -35.2592424], 'decimalLatitude':-35.2592424, 'decimalLongitude':149.0651439], 'source':'point'], 'projects':[], 'type':'surveyArea']
 
         String userId = 'u1'
 

src/test/groovy/au/org/ala/ecodata/SiteServiceSpec.groovy

@@ -146,6 +146,23 @@ class SiteServiceSpec extends MongoSpec implements ServiceUnitTest<SiteService>
         Site.findBySiteId(result.siteId).name == 'Site 1'
     }
 
+    def "An externalId can be supplied and the Site will convert it to the correct format"() {
+        when:
+        def result
+        Site.withSession { session ->
+            result = service.create([name:'Site 1', externalId:'e1'])
+            session.flush()
+        }
+        then:
+        def site = Site.findBySiteId(result.siteId)
+        site.name == 'Site 1'
+        site.externalId == 'e1'
+        site.externalIds.size() == 1
+        site.externalIds[0].externalId == 'e1'
+        site.externalIds[0].idType == ExternalId.IdType.UNSPECIFIED
+    }
+
+
     def "A new site should not allow the siteId to be supplied"() {
         when:
         def result

src/test/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfigSpec.groovy

@@ -1,8 +1,8 @@
 package au.org.ala.ecodata.paratoo
 
+
 import grails.converters.JSON
 import groovy.json.JsonSlurper
-import org.geotools.geojson.geom.GeometryJSON
 import org.grails.web.converters.marshaller.json.CollectionMarshaller
 import org.grails.web.converters.marshaller.json.MapMarshaller
 import spock.lang.Specification