From b51483a1d816550f47cf861829f80ed760dfde55 Mon Sep 17 00:00:00 2001
From: Adam Collins <adam.collins@csiro.au>
Date: Thu, 16 Nov 2023 13:46:03 +1000
Subject: [PATCH 01/13] bump 5.1.0-SNAPSHOT

---
 build.gradle | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build.gradle b/build.gradle
index 8794a08c..469bb7bf 100644
--- a/build.gradle
+++ b/build.gradle
@@ -1,5 +1,5 @@
 buildscript {
-    version "5.0.0-SNAPSHOT"
+    version "5.1.0-SNAPSHOT"
     group "au.org.ala"
 }
 

From e7f8f35530ac4582deec8b0261792118ca83e07b Mon Sep 17 00:00:00 2001
From: Adam Collins <adam.collins@csiro.au>
Date: Mon, 20 Nov 2023 12:48:33 +1000
Subject: [PATCH 02/13] openapi

---
 .../org/ala/collectory/DataController.groovy  | 240 +++++++++++-------
 .../ala/collectory/LookupController.groovy    |  10 +-
 2 files changed, 153 insertions(+), 97 deletions(-)

diff --git a/grails-app/controllers/au/org/ala/collectory/DataController.groovy b/grails-app/controllers/au/org/ala/collectory/DataController.groovy
index cb2c0aac..e2c396a5 100644
--- a/grails-app/controllers/au/org/ala/collectory/DataController.groovy
+++ b/grails-app/controllers/au/org/ala/collectory/DataController.groovy
@@ -35,9 +35,9 @@ import static io.swagger.v3.oas.annotations.enums.ParameterIn.QUERY
 class DataController {
 
     def crudService, emlRenderService, collectoryAuthService, metadataService, providerGroupService,
-            activityLogService, asyncGbifRegistryService
+        activityLogService, asyncGbifRegistryService
 
-    def index = { }
+    def index = {}
 
     private def check(params) {
         def uid = params.uid
@@ -54,11 +54,11 @@ class DataController {
                 }
             } else {
 
-                if (params.entity){
+                if (params.entity) {
                     params.pg = providerGroupService._get(params.uid, params.entity)
                 }
 
-                if (!params.pg){
+                if (!params.pg) {
                     // doesn't exist
                     notFound "no entity with uid = ${uid}"
                     return false
@@ -87,14 +87,14 @@ class DataController {
                 return false
             }
             // inject the user name into the session so it can be used by audit logging if changes are made
-            session.username = params?.json?.user?: collectoryAuthService.username()
+            session.username = params?.json?.user ?: collectoryAuthService.username()
         }
         return true
     }
 
     /******* Web Services Catalogue *******/
 
-    def catalogue = { }
+    def catalogue = {}
 
     /***** CRUD RESTful services ********/
 
@@ -104,8 +104,8 @@ class DataController {
     final static String RFC1123_PATTERN = "EEE, dd MMM yyyy HH:mm:ss z";
 
     /**
-      * DateFormat to be used to format dates
-      */
+     * DateFormat to be used to format dates
+     */
     final static DateFormat rfc1123Format = new SimpleDateFormat(RFC1123_PATTERN)
     static {
         rfc1123Format.setTimeZone(TimeZone.getTimeZone("GMT"));
@@ -135,10 +135,10 @@ class DataController {
         if (eTag) {
             addETagHeader eTag
         }
-        render(text:content, encoding:"UTF-8", contentType: "application/json")
+        render(text: content, encoding: "UTF-8", contentType: "application/json")
     }
 
-    def renderAsJson = {json, last, eTag ->
+    def renderAsJson = { json, last, eTag ->
         renderJson(json as JSON, last, eTag)
     }
 
@@ -150,17 +150,17 @@ class DataController {
         response.addHeader 'content-location', grailsApplication.config.grails.serverURL + relativeUri
     }
 
-    def created = {clazz, uid ->
+    def created = { clazz, uid ->
         addLocation "/ws/${clazz}/${uid}"
-        render(status:201, text:'inserted entity')
+        render(status: 201, text: 'inserted entity')
     }
 
-    def badRequest = {text ->
-        render(status:400, text: text)
+    def badRequest = { text ->
+        render(status: 400, text: text)
     }
 
     def success = { text ->
-        render(status:200, text: text)
+        render(status: 200, text: text)
     }
 
     def notModified = {
@@ -168,26 +168,26 @@ class DataController {
     }
 
     def notFound = { text ->
-        render(status:404, text: text)
+        render(status: 404, text: text)
     }
 
     def return404() {
-        render(status:404)
+        render(status: 404)
     }
 
     def notAllowed = {
-        response.addHeader 'allow','POST'
-        render(status:405, text: 'Only POST supported')
+        response.addHeader 'allow', 'POST'
+        render(status: 405, text: 'Only POST supported')
     }
 
     def unauthorised = {
         // using the 'forbidden' response code here as 401 causes the client to ask for a log in
-        render(status:403, text: 'You are not authorised to use this service')
+        render(status: 403, text: 'You are not authorised to use this service')
     }
 
     def noApiKey = {
         // using the 'forbidden' response code here as 401 causes the client to ask for a log in
-        render(status:400, text: 'This service requires API key')
+        render(status: 400, text: 'This service requires API key')
     }
 
     /**
@@ -210,7 +210,7 @@ class DataController {
         switch (word?.size()) {
             case 0: return ""
             case 1: return word[0].toUpperCase()
-            default: return word[0].toUpperCase() + word [1..-1]
+            default: return word[0].toUpperCase() + word[1..-1]
         }
     }
 
@@ -239,7 +239,7 @@ class DataController {
                             in = PATH,
                             description = "entity i.e.  collection, institution, dataProvider, dataResource, tempDataResource, dataHub",
                             schema = @Schema(implementation = String),
-                            example ="collection",
+                            example = "collection",
                             required = true
                     ),
                     @Parameter(
@@ -282,7 +282,7 @@ class DataController {
     )
     @Path("/ws/{entity}/{uid}")
     @Produces("application/json")
-    def saveEntity () {
+    def saveEntity() {
         def ok = check(params)
         if (ok) {
             def pg = params.pg
@@ -319,8 +319,8 @@ class DataController {
     /**
      * Define some variations on the level of detail returned for lists.
      */
-    def brief = {[name: it.name, uri: it.buildUri(), uid: it.uid]}
-    def summary = {[name: it.name, uri: it.buildUri(), uid: it.uid, logo: it.buildLogoUrl()]}
+    def brief = { [name: it.name, uri: it.buildUri(), uid: it.uid] }
+    def summary = { [name: it.name, uri: it.buildUri(), uid: it.uid, logo: it.buildLogoUrl()] }
 
     def index() {
         def root = params.root
@@ -339,13 +339,13 @@ class DataController {
     }
 
     def serveFile = {
-        def dirpath =  "/" + params.directory + "/"
+        def dirpath = "/" + params.directory + "/"
         def uri = URLDecoder.decode(request.forwardURI, "UTF-8")
         def idx = uri.lastIndexOf(dirpath) + dirpath.length()
         def fullFileName = uri.substring(idx)
         def file = new File(grailsApplication.config.repository.location.images + File.separator + params.directory, fullFileName)
-        if(file.exists()){
-            if(fullFileName.endsWith(".json")){
+        if (file.exists()) {
+            if (fullFileName.endsWith(".json")) {
                 response.setContentType("application/json")
             }
             file.withInputStream { response.outputStream << it }
@@ -355,14 +355,14 @@ class DataController {
     }
 
     def fileDownload = {
-        def dirpath =  "/" + params.directory + "/"
+        def dirpath = "/" + params.directory + "/"
         def idx = request.forwardURI.lastIndexOf(dirpath) + dirpath.length()
         def fullFileName = request.forwardURI.substring(idx)
         def file = new File(grailsApplication.config.uploadFilePath + File.separator + params.directory, fullFileName)
-        if (!file.exists()){
+        if (!file.exists()) {
             file = new File(grailsApplication.config.uploadFilePath + File.separator + params.directory, URLDecoder.decode(fullFileName, "UTF-8"))
         }
-        if (file.exists()){
+        if (file.exists()) {
             //set the content type
             response.setContentType("application/octet-stream")
             response.setHeader("Content-disposition", "attachment;filename=" + file.getName())
@@ -393,20 +393,27 @@ class DataController {
             description = "Get detailed information for a specific entity",
             parameters = [
                     @Parameter(
-                        name = "entity",
-                        in = PATH,
-                        description = "entity type -  e.g. datResource, dataProvider etc",
-                        schema = @Schema(implementation = String),
-                        example = "collection",
-                        required = true
+                            name = "entity",
+                            in = PATH,
+                            description = "entity type; collection, institution, dataProvider, dataResource, tempDataResource, dataHub",
+                            schema = @Schema(implementation = String),
+                            example = "collection",
+                            required = true
                     ),
                     @Parameter(
-                        name = "uid",
-                        in = PATH,
-                        description = "uid of an instance of entity",
-                        schema = @Schema(implementation = String),
-                        example = "co43",
-                        required = true
+                            name = "uid",
+                            in = PATH,
+                            description = "uid of an instance of entity",
+                            schema = @Schema(implementation = String),
+                            example = "co43",
+                            required = true
+                    ),
+                    @Parameter(
+                            name = "apikey",
+                            in = HEADER,
+                            description = "authorisation for dataResource connection details",
+                            schema = @Schema(implementation = String),
+                            required = false
                     )
             ],
             responses = [
@@ -416,7 +423,7 @@ class DataController {
                             content = [
                                     @Content(
                                             mediaType = "application/json",
-                                            array = @ArraySchema(schema = @Schema(implementation = Entity))
+                                            schema = @Schema(oneOf = [Collection, Institution, DataProvider, DataResource, TempDataResource, DataHub])
                                     )
                             ],
                             headers = [
@@ -435,7 +442,7 @@ class DataController {
      * The functionality described above has been preserved to maintain backwards compatibility but should be removed in the future once the legacy API key access is deprecated
      */
 
-    def getEntity () {
+    def getEntity() {
         check(params)
         if (params.entity == 'tempDataResource') {
             forward(controller: 'tempDataResource', action: 'getEntity')
@@ -450,7 +457,7 @@ class DataController {
                 if (clazz == 'DataResource') {
                     // this auth check (JWT or API key) is a special case handling to support backwards compatibility(which used to check for API key).
                     String requiredRoles = grailsApplication.config.ROLE_ADMIN
-                    def authCheck = collectoryAuthService.isAuthorisedWsRequest(getParams(), request, response, requiredRoles,null)
+                    def authCheck = collectoryAuthService.isAuthorisedWsRequest(getParams(), request, response, requiredRoles, null)
                     entityInJson = crudService."read${clazz}"(params.pg, authCheck)
                 } else {
                     entityInJson = crudService."read${clazz}"(params.pg)
@@ -480,6 +487,46 @@ class DataController {
         }
     }
 
+    @Operation(
+            method = "GET",
+            tags = "collection, institution, dataProvider, dataResource, tempDataResource, dataHub",
+            operationId = "listEntities",
+            summary = "list entities for a given type",
+            parameters = [
+                    @Parameter(
+                            name = "entity",
+                            in = PATH,
+                            description = "entity type; collection, institution, dataProvider, dataResource, tempDataResource, dataHub",
+                            schema = @Schema(implementation = String),
+                            example = "collection",
+                            required = true
+                    )
+            ],
+            responses = [
+                    @ApiResponse(
+                            description = "List of entities",
+                            responseCode = "200",
+                            content = [
+                                    @Content(
+                                            mediaType = "application/json",
+                                            array = @ArraySchema(schema = @Schema(implementation = Entity))
+                                    )
+                            ],
+                            headers = [
+                                    @Header(name = 'Access-Control-Allow-Headers', description = "CORS header", schema = @Schema(type = "String")),
+                                    @Header(name = 'Access-Control-Allow-Methods', description = "CORS header", schema = @Schema(type = "String")),
+                                    @Header(name = 'Access-Control-Allow-Origin', description = "CORS header", schema = @Schema(type = "String"))
+                            ]
+                    )
+            ],
+            security = []
+    )
+    @Path("/ws/{entity}")
+    @Produces("application/json")
+    def listEntity() {
+        // dummy method for openapi definition
+    }
+
     @JsonIgnoreProperties('metaClass')
     class Entity {
         String name
@@ -518,8 +565,7 @@ class DataController {
                 def value = it[params.groupBy]
                 if (groups[value]) {
                     groups[value]++
-                }
-                else {
+                } else {
                     groups[value] = 1
                 }
             }
@@ -531,6 +577,7 @@ class DataController {
 
         renderAsJson results, last, ""
     }
+
     @Operation(
             method = "GET",
             tags = "gbif",
@@ -557,7 +604,7 @@ class DataController {
     )
     @Path("/ws/syncGBIF")
     @Produces("application/json")
-    def syncGBIF () {
+    def syncGBIF() {
         asyncGbifRegistryService.updateAllResources()
                 .onComplete {
                     log.info "Sync complete"
@@ -668,18 +715,18 @@ class DataController {
      */
     def delete = {
         if (grailsApplication.config.deletesForbidden) {
-            render(status:405, text:'delete is currently unavailable')
+            render(status: 405, text: 'delete is currently unavailable')
             return
         }
         // check role
         if (params.uid) {
             def pg = params.uid.startsWith('drt') ?
-                TempDataResource.findByUid(params.uid) :
-                providerGroupService._get(params.uid)
+                    TempDataResource.findByUid(params.uid) :
+                    providerGroupService._get(params.uid)
             if (pg) {
                 def name = pg.name
                 pg.delete()
-                def message = ['message':"deleted ${name}"]
+                def message = ['message': "deleted ${name}"]
                 render message as JSON
             } else {
                 def error = ['error': "no uid specified"]
@@ -725,7 +772,7 @@ class DataController {
                             in = QUERY,
                             description = "restrict to associated object names that contain this value",
                             schema = @Schema(implementation = String),
-                            example ="Environment",
+                            example = "Environment",
                             required = false
                     )
             ],
@@ -750,7 +797,7 @@ class DataController {
     )
     @Path("/ws/eml/{id}")
     @Produces("text/xml")
-    def eml () {
+    def eml() {
         if (params.id) {
             def pg = providerGroupService._get(params.id)
             if (pg) {
@@ -783,7 +830,6 @@ class DataController {
     }
 
 
-
     def validate(xml) {
         try {
             def factory = SchemaFactory.newInstance(XMLConstants.W3C_XML_SCHEMA_NS_URI)
@@ -906,7 +952,7 @@ class DataController {
     )
     @Path("/ws/contacts/{id}")
     @Produces("application/json")
-    def contacts () {
+    def contacts() {
         if (params.id) {
             def c = Contact.get(params.id)
             if (c) {
@@ -914,9 +960,9 @@ class DataController {
                 addVaryAcceptHeader()
                 def cm = buildContactModel(c)
                 withFormat {
-                    json {render cm as JSON}
-                    csv {render (contentType: 'text/csv', text: CONTACT_HEADER + mapToCsv(cm))}
-                    xml {render (contentType: 'text/xml', text: objToXml(cm, 'contact'))}
+                    json { render cm as JSON }
+                    csv { render(contentType: 'text/csv', text: CONTACT_HEADER + mapToCsv(cm)) }
+                    xml { render(contentType: 'text/xml', text: objToXml(cm, 'contact')) }
                 }
             } else {
                 badRequest ' no such id'
@@ -925,25 +971,27 @@ class DataController {
             addContentLocation "/ws/contacts"
             addVaryAcceptHeader()
             withFormat {
-                json {render Contact.list().collect { buildContactModel(it) } as JSON}
-                csv {render (contentType: 'text/csv',
-                        text: CONTACT_HEADER + Contact.list().collect { mapToCsv(buildContactModel(it)) }.join(''))}
-                xml {render (contentType: 'text/xml', text: objToXml(Contact.list().collect { buildContactModel(it) }, 'contacts'))}
+                json { render Contact.list().collect { buildContactModel(it) } as JSON }
+                csv {
+                    render(contentType: 'text/csv',
+                            text: CONTACT_HEADER + Contact.list().collect { mapToCsv(buildContactModel(it)) }.join(''))
+                }
+                xml { render(contentType: 'text/xml', text: objToXml(Contact.list().collect { buildContactModel(it) }, 'contacts')) }
             }
         }
     }
 
     def buildContactModel(contact) {
         return new LinkedHashMap(
-            [title: contact.title, firstName: contact.firstName, lastName: contact.lastName, email: contact.email, phone: contact.phone,
-             fax: contact.fax, mobile: contact.mobile, publish: contact.publish, dateCreated: contact.dateCreated, lastUpdated: contact.lastUpdated])
+                [title: contact.title, firstName: contact.firstName, lastName: contact.lastName, email: contact.email, phone: contact.phone,
+                 fax  : contact.fax, mobile: contact.mobile, publish: contact.publish, dateCreated: contact.dateCreated, lastUpdated: contact.lastUpdated])
     }
 
     def buildContactForModel(cf, urlContext) {
         return new LinkedHashMap(
-            [contact: buildContactModel(cf.contact), role: cf.role, primaryContact: cf.primaryContact,
-                    editor: cf.administrator, notify: cf.notify, dateCreated: cf.dateCreated, lastUpdated: cf.dateLastModified,
-                    uri: "${grailsApplication.config.grails.serverURL}/ws/${urlContext}/${cf.entityUid}/contacts/${cf.id}"])
+                [contact: buildContactModel(cf.contact), role: cf.role, primaryContact: cf.primaryContact,
+                 editor : cf.administrator, notify: cf.notify, dateCreated: cf.dateCreated, lastUpdated: cf.dateLastModified,
+                 uri    : "${grailsApplication.config.grails.serverURL}/ws/${urlContext}/${cf.entityUid}/contacts/${cf.id}"])
     }
 
     /**
@@ -1015,9 +1063,9 @@ class DataController {
     )
     @Path("/ws/contacts/{id}")
     @Produces("application/json")
-    def updateContact () {
+    def updateContact() {
         def ok = check(params)
-        if (!ok){
+        if (!ok) {
             return
         }
         def props = params.json
@@ -1031,7 +1079,7 @@ class DataController {
                 Contact.withTransaction {
                     c.save(flush: true)
                 }
-                c.errors.each {  log.error(it.toString()) }
+                c.errors.each { log.error(it.toString()) }
                 addContentLocation "/ws/contacts/${c.id}"
                 def cm = buildContactModel(c)
                 cm.id = c.id
@@ -1074,7 +1122,7 @@ class DataController {
     }
 
     @Transactional
-    def deleteContact () {
+    def deleteContact() {
         if (params.id) {
             // update
             def c = Contact.get(params.id)
@@ -1112,17 +1160,17 @@ class DataController {
         addContentLocation "/ws/${params.entity}/${params.pg.uid}/contacts"
         addVaryAcceptHeader()
         withFormat {
-            json {render contactList as JSON}
+            json { render contactList as JSON }
             csv {
                 def out = new StringWriter()
                 out << "name, role, primary contact, editor, notify, email, phone\n"
                 contactList.each {
-                    out << "\"${(it.contact.firstName + ' ' + it.contact.lastName).trim()}\",\"${it.role}\",${it.primaryContact},${it.editor},${it.notify},${it.contact.email?:""},${it.contact.phone?:""}\n"
+                    out << "\"${(it.contact.firstName + ' ' + it.contact.lastName).trim()}\",\"${it.role}\",${it.primaryContact},${it.editor},${it.notify},${it.contact.email ?: ""},${it.contact.phone ?: ""}\n"
                 }
                 response.addHeader "Content-Type", "text/csv"
                 render out.toString()
             }
-            xml {render (contentType: 'text/xml', text: objToXml(contactList, 'contactFors'))}
+            xml { render(contentType: 'text/xml', text: objToXml(contactList, 'contactFors')) }
         }
     }
 
@@ -1156,10 +1204,10 @@ class DataController {
                     xml { render(contentType: 'text/xml', text: objToXml(cm, 'contactFor')) }
                 }
             } else {
-                forward(action:'contactsForEntity')
+                forward(action: 'contactsForEntity')
             }
         } else {
-            forward(action:'contactsForEntity')
+            forward(action: 'contactsForEntity')
         }
     }
 
@@ -1172,7 +1220,7 @@ class DataController {
      */
     def contactsForEntities = {
         def domain = grailsApplication.getClassForName("au.org.ala.collectory.${capitalise(params.entity)}")
-        def model = buildContactsModel(domain.list([sort:'name']))
+        def model = buildContactsModel(domain.list([sort: 'name']))
         addContentLocation "/ws/${params.entity}/contacts"
         addVaryAcceptHeader()
         withFormat {
@@ -1180,10 +1228,10 @@ class DataController {
                 render model as JSON
             }
             csv {
-                render (contentType: 'text/csv',
+                render(contentType: 'text/csv',
                         text: SHORT_CONTACTS_HEADER + listToCsv(model))
             }
-            xml {render (contentType: 'text/xml', text: objToXml(model, 'contacts'))}
+            xml { render(contentType: 'text/xml', text: objToXml(model, 'contacts')) }
         }
     }
 
@@ -1210,8 +1258,7 @@ class DataController {
         def contact = Contact.get(params.id)
         if (!contact) {
             badRequest "contact ${params.id} does not exist"
-        }
-        else {
+        } else {
             def result = collectoryAuthService?.authorisedForUser(contact)
             renderAsJson result.sorted, result.latestMod, result.keys.toString().encodeAsMD5()
         }
@@ -1240,7 +1287,7 @@ class DataController {
      */
     def notification() {
         def ok = check(params)
-        if (!ok){
+        if (!ok) {
             return
         }
         //println "notify"
@@ -1352,9 +1399,9 @@ class DataController {
     )
     @Path("/ws/{entity}/{uid}/contacts/{id}")
     @Produces("application/json")
-    def updateContactFor () {
+    def updateContactFor() {
         def ok = check(params)
-        if (!ok){
+        if (!ok) {
             return
         }
         def props = params.json
@@ -1368,7 +1415,7 @@ class DataController {
             Contact.withTransaction {
                 c.save(flush: true)
             }
-            c.errors.each {  log.error(it.toString()) }
+            c.errors.each { log.error(it.toString()) }
             success 'updated'
         } else {
             // create
@@ -1385,14 +1432,14 @@ class DataController {
         }
     }
 
-    def deleteContactFor () {
+    def deleteContactFor() {
         def ok = check(params)
-        if (!ok){
+        if (!ok) {
             return
         }
         def props = params.json
         props.userLastModified = session.username
-        log.error("body = "  + props)
+        log.error("body = " + props)
         def c = Contact.get(params.id)
         def cf = ContactFor.findByContactAndEntityUid(c, params.pg.uid)
         if (cf) {
@@ -1450,7 +1497,7 @@ class DataController {
                 }
             }
         }
-        render(contentType: 'text/csv', text:csv)
+        render(contentType: 'text/csv', text: csv)
     }
 
     private String encodeHints(hints) {
@@ -1470,12 +1517,13 @@ class DataController {
         def writer = new StringWriter()
         MarkupBuilder xml = new MarkupBuilder(writer)
         xml."${root}" {
-            toXml(obj,xml, (root[-1] == 's') ? root[0..-2] : 'item')
+            toXml(obj, xml, (root[-1] == 's') ? root[0..-2] : 'item')
         }
         return writer.toString()
     }
 
     /* called recursively to build xml */
+
     def toXml(obj, xml, listElement) {
         if (obj instanceof List) {
             obj.each { item ->
@@ -1484,7 +1532,7 @@ class DataController {
         } else {
             obj.each { key, value ->
                 if (value && value instanceof Map) {
-                    xml."${key}" {toXml(value, xml, listElement)}
+                    xml."${key}" { toXml(value, xml, listElement) }
                 } else {
                     xml."${key}"(value)
                 }
@@ -1507,8 +1555,8 @@ class DataController {
      */
     def mapToCsv(map) {
         def out = new StringWriter()
-        def list = map.collect {key, value -> value}
-        list.eachWithIndex {it, idx ->
+        def list = map.collect { key, value -> value }
+        list.eachWithIndex { it, idx ->
             out << toCsvItem(it)
             if (idx == list.size() - 1) {
                 out << '\n'
diff --git a/grails-app/controllers/au/org/ala/collectory/LookupController.groovy b/grails-app/controllers/au/org/ala/collectory/LookupController.groovy
index b0ed391f..e98289d8 100644
--- a/grails-app/controllers/au/org/ala/collectory/LookupController.groovy
+++ b/grails-app/controllers/au/org/ala/collectory/LookupController.groovy
@@ -300,6 +300,15 @@ class LookupController {
             tags = "citations",
             operationId = "getCitations",
             summary = "Get citations for a list of data resource UIDs.",
+            parameters = [
+                    @Parameter(
+                            name = "Accept",
+                            in = HEADER,
+                            schema = @Schema(implementation = String),
+                            example = "application/json or text/csv or text/plain",
+                            required = false
+                    )
+            ],
             requestBody = @RequestBody(
                     required = true,
                     description = "A JSON array containing dataResource UIDs.",
@@ -361,7 +370,6 @@ class LookupController {
             security = []
     )
     @Path("/ws/citations")
-    @Produces("application/json")
     def citations() {
         if (params.include) {
             params.uids = "[${params.include}]"

From a1b700037361ec64fd8bdb181d133cd0b5f00f47 Mon Sep 17 00:00:00 2001
From: Adam Collins <adam.collins@csiro.au>
Date: Tue, 21 Nov 2023 15:23:53 +1000
Subject: [PATCH 03/13] cache DataHub.list

---
 .../collectory/CollectionController.groovy    | 15 ++--
 .../ala/collectory/DataHubController.groovy   |  7 ++
 .../ala/collectory/DataLinkController.groovy  |  4 +-
 .../ala/collectory/LookupController.groovy    | 40 +++++++++-
 .../collectory/ProviderGroupController.groovy |  6 --
 .../collectory/ProviderMapController.groovy   |  1 -
 .../ala/collectory/PublicController.groovy    |  9 ---
 .../ala/collectory/ReportsController.groovy   | 12 +--
 .../ala/collectory/TempDataInterceptor.groovy |  1 -
 .../au/org/ala/collectory/Collection.groovy   | 54 +-------------
 .../au/org/ala/collectory/DataResource.groovy | 49 +------------
 .../au/org/ala/collectory/Institution.groovy  | 40 ----------
 .../ala/collectory/ActivityLogService.groovy  |  6 +-
 .../ala/collectory/CollectionService.groovy   | 69 ++++++++++++++++++
 .../collectory/CollectoryAuthService.groovy   | 10 ++-
 .../au/org/ala/collectory/CrudService.groovy  | 25 +++----
 .../org/ala/collectory/DataHubService.groovy  | 27 +++++++
 .../ala/collectory/DataLoaderService.groovy   | 73 +------------------
 .../ala/collectory/DataResourceService.groovy | 63 ++++++++++++++++
 .../ala/collectory/InstitutionService.groovy  | 57 +++++++++++++++
 .../org/ala/collectory/MetadataService.groovy |  1 +
 .../collectory/ProviderGroupService.groovy    |  1 -
 .../ala/collectory/CollectoryTagLib.groovy    |  7 +-
 .../org/ala/collectory/Classification.groovy  |  1 -
 .../org/ala/collectory/GBIFLoadSummary.groovy |  2 -
 .../org/ala/collectory/ProviderGroup.groovy   | 11 ---
 26 files changed, 304 insertions(+), 287 deletions(-)
 create mode 100644 grails-app/services/au/org/ala/collectory/CollectionService.groovy
 create mode 100644 grails-app/services/au/org/ala/collectory/DataHubService.groovy
 create mode 100644 grails-app/services/au/org/ala/collectory/DataResourceService.groovy
 create mode 100644 grails-app/services/au/org/ala/collectory/InstitutionService.groovy

diff --git a/grails-app/controllers/au/org/ala/collectory/CollectionController.groovy b/grails-app/controllers/au/org/ala/collectory/CollectionController.groovy
index 0f0f7d69..8c1fdd24 100644
--- a/grails-app/controllers/au/org/ala/collectory/CollectionController.groovy
+++ b/grails-app/controllers/au/org/ala/collectory/CollectionController.groovy
@@ -1,7 +1,6 @@
 package au.org.ala.collectory
 
 import grails.converters.JSON
-import grails.validation.Validateable
 import org.springframework.validation.annotation.Validated
 
 import java.text.NumberFormat
@@ -15,6 +14,8 @@ class CollectionController extends ProviderGroupController {
     }
 
     def idGeneratorService
+    def collectionService
+    def institutionService
 
     def index = {
         redirect(action:"list")
@@ -113,9 +114,8 @@ class CollectionController extends ProviderGroupController {
      */
     def summary = {
         Collection collectionInstance = findCollection(params.id)
-        println ">> summary id = " + params.id
         if (collectionInstance) {
-            render collectionInstance.buildSummary() as JSON
+            render collectionService.buildSummary(collectionInstance.uid) as JSON
         } else {
             log.error "Unable to find collection for id = ${params.id}"
             def error = ["error":"unable to find collection for id = " + params.id]
@@ -143,7 +143,12 @@ class CollectionController extends ProviderGroupController {
         }
         ProviderGroup pg = ProviderMap.findMatch(inst, coll)
         if (pg) {
-            render pg.buildSummary() as JSON
+            if (pg[0..1] == 'co') {
+                render collectionService.buildSummary(pg.uid) as JSON
+            } else {
+                // institution
+                render institutionService.buildSummary(pg.uid) as JSON
+            }
         } else {
             def error = ["error":"unable to find collection with inst code = ${inst} and coll code = ${coll}"]
             render error as JSON
@@ -205,7 +210,7 @@ class CollectionController extends ProviderGroupController {
         } else {
             collection.collectionType = (params.collectionType as JSON).toString()
         }
-        
+
         params.remove('collectionType')
 
         // special handling for keywords
diff --git a/grails-app/controllers/au/org/ala/collectory/DataHubController.groovy b/grails-app/controllers/au/org/ala/collectory/DataHubController.groovy
index 34bc3611..e268ba69 100644
--- a/grails-app/controllers/au/org/ala/collectory/DataHubController.groovy
+++ b/grails-app/controllers/au/org/ala/collectory/DataHubController.groovy
@@ -2,6 +2,8 @@ package au.org.ala.collectory
 
 class DataHubController extends ProviderGroupController {
 
+    def grailsCacheAdminService
+
     DataHubController() {
         entityName = "DataHub"
         entityNameLower = "dataHub"
@@ -22,6 +24,9 @@ class DataHubController extends ProviderGroupController {
     }
 
     def show = {
+        // After updating members this page is shown
+        grailsCacheAdminService.clearCache('dataHubCache')
+
         def instance = get(params.id)
         if (!instance) {
             flash.message = "${message(code: 'default.not.found.message', args: [message(code: 'dataHub.label', default: 'Data Hub'), params.id])}"
@@ -36,6 +41,8 @@ class DataHubController extends ProviderGroupController {
     }
 
     def delete = {
+        grailsCacheAdminService.clearCache('dataHubCache')
+
         def instance = get(params.id)
         if (instance) {
             if (isAdmin()) {
diff --git a/grails-app/controllers/au/org/ala/collectory/DataLinkController.groovy b/grails-app/controllers/au/org/ala/collectory/DataLinkController.groovy
index 1f47f041..5b992731 100644
--- a/grails-app/controllers/au/org/ala/collectory/DataLinkController.groovy
+++ b/grails-app/controllers/au/org/ala/collectory/DataLinkController.groovy
@@ -22,8 +22,6 @@ class DataLinkController {
     }
 
     def create = {
-        println "consumer = ${params.consumer}"
-        println "provider = ${params.provider}"
         def dataLinkInstance = new DataLink()
         dataLinkInstance.properties = params
         return [dataLinkInstance: dataLinkInstance, returnTo: params.returnTo]
@@ -68,7 +66,7 @@ class DataLinkController {
             if (params.version) {
                 def version = params.version.toLong()
                 if (dataLinkInstance.version > version) {
-                    
+
                     dataLinkInstance.errors.rejectValue("version", "default.optimistic.locking.failure", [message(code: 'dataLink.label', default: 'DataLink')] as Object[], "Another user has updated this DataLink while you were editing")
                     render(view: "edit", model: [dataLinkInstance: dataLinkInstance, returnTo: params.returnTo])
                     return
diff --git a/grails-app/controllers/au/org/ala/collectory/LookupController.groovy b/grails-app/controllers/au/org/ala/collectory/LookupController.groovy
index e98289d8..8cabae4e 100644
--- a/grails-app/controllers/au/org/ala/collectory/LookupController.groovy
+++ b/grails-app/controllers/au/org/ala/collectory/LookupController.groovy
@@ -29,6 +29,9 @@ class LookupController {
 
     def idGeneratorService
     def providerGroupService
+    def dataResourceService
+    def collectionService
+    def institutionService
 
     static allowedMethods = [citation:['POST','GET']]
 
@@ -97,7 +100,12 @@ class LookupController {
             }
             ProviderGroup pg = ProviderMap.findMatch(inst, coll)
             if (pg) {
-                render pg.buildSummary() as JSON
+                if (pg.uid[0..1] == 'co') {
+                    render collectionService.buildSummary(pg.uid) as JSON
+                } else {
+                    // institution
+                    render institutionService.buildSummary(pg.uid) as JSON
+                }
             } else {
                 def error = ["error":"unable to find collection with inst code = ${inst} and coll code = ${coll}"]
                 render error as JSON
@@ -136,7 +144,15 @@ class LookupController {
             // summary for single entity
             pg = providerGroupService._get(uid)
             if (pg) {
-                render pg.buildSummary() as JSON
+                if (pg.uid[0..1] == 'co') {
+                    render collectionService.buildSummary(pg.uid) as JSON
+                } else if (pg[0..1] == 'in') {
+                    render institutionService.buildSummary(pg.uid) as JSON
+                } else if (pg[0..1] == 'dr') {
+                    render dataResourceService.buildSummary(pg.uid) as JSON
+                } else {
+                    render pg.buildSummary() as JSON
+                }
             } else {
                 log.error "Unable to find entity. id = ${params.id}"
                 def error = ["error":"unable to find entity - id = ${params.id}"]
@@ -146,7 +162,15 @@ class LookupController {
             // return summaries for all
             def list = []
             domain.list().each {
-                list << it.buildSummary()
+                if (it.uid[0..1] == 'co') {
+                    list << collectionService.buildSummary(it.uid)
+                } else if (pg[0..1] == 'in') {
+                    list << institutionService.buildSummary(it.uid)
+                } else if (pg[0..1] == 'dr') {
+                    list << dataResourceService.buildSummary(it.uid)
+                } else {
+                    list << it.buildSummary()
+                }
             }
             render list as JSON
         }
@@ -209,7 +233,15 @@ class LookupController {
             instance = findCollection(params.id)
         }
         if (instance) {
-            render instance.buildSummary() as JSON
+            if (instance.uid[0..1] == 'co') {
+                render collectionService.buildSummary(instance.uid) as JSON
+            } else if (pg[0..1] == 'in') {
+                render institutionService.buildSummary(instance.uid) as JSON
+            } else if (pg[0..1] == 'dr') {
+                render dataResourceService.buildSummary(instance.uid) as JSON
+            } else {
+                render instance.buildSummary() as JSON
+            }
         } else {
             // we would normally log this as a warning but it occurs too often
             //log.warning "Unable to find entity for id = ${params.id}"
diff --git a/grails-app/controllers/au/org/ala/collectory/ProviderGroupController.groovy b/grails-app/controllers/au/org/ala/collectory/ProviderGroupController.groovy
index 8d24d5f3..ad93f461 100644
--- a/grails-app/controllers/au/org/ala/collectory/ProviderGroupController.groovy
+++ b/grails-app/controllers/au/org/ala/collectory/ProviderGroupController.groovy
@@ -66,11 +66,9 @@ abstract class ProviderGroupController {
         if (params.version) {
             def version = params.version.toLong()
             if (pg.version > version) {
-                println "db version = ${pg.version} submitted version = ${version}"
                 pg.errors.rejectValue("version", "default.optimistic.locking.failure",
                         [message(code: "${pg.urlForm()}.label", default: pg.entityType())] as Object[],
                         message(code: "provider.group.controller.02", default: "Another user has updated this") + " ${pg.entityType()} " + message(code: "provider.group.controller.03", default: "while you were editing. This page has been refreshed with the current values."))
-                println "error added - rendering ${view}"
                 response.setHeader("Content-type", "text/plain; charset=UTF-8")
                 render(view: view, model: [command: pg])
             }
@@ -192,7 +190,6 @@ abstract class ProviderGroupController {
             if (c.hasErrors()) {
                 c.errors.each {
                     log.debug("Error saving new contact for ${user} - ${it}")
-                    println "Error saving new contact for ${user} - ${it}"
                 }
             }
         }
@@ -200,7 +197,6 @@ abstract class ProviderGroupController {
     }
 
     def cancel = {
-        //println "Cancel - returnTo = ${params.returnTo}"
         if (params.returnTo) {
             redirect(uri: params.returnTo)
         } else {
@@ -381,7 +377,6 @@ abstract class ProviderGroupController {
             def th = pg.taxonomyHints ? JSON.parse(pg.taxonomyHints) : [:]
             th.range = rangeList
             pg.taxonomyHints = th as JSON
-            println pg.taxonomyHints
 
             pg.userLastModified = collectoryAuthService?.username()
             if (!pg.hasErrors()) {
@@ -708,7 +703,6 @@ abstract class ProviderGroupController {
                     file.transferTo(f)
                     activityLogService.log collectoryAuthService?.username(), collectoryAuthService?.userInRole(grailsApplication.config.ROLE_ADMIN), Action.UPLOAD_IMAGE, filename
                 } else {
-                    println "reject file of size ${file.size}"
                     pg.errors.rejectValue('imageRef', 'image.too.big', message(code: "provider.group.controller.13", default: "The image you selected is too large. Images are limited to 200KB."))
                     render(view: "/shared/images", model: [command: pg, target: target])
                     return
diff --git a/grails-app/controllers/au/org/ala/collectory/ProviderMapController.groovy b/grails-app/controllers/au/org/ala/collectory/ProviderMapController.groovy
index 08328530..33164a8d 100644
--- a/grails-app/controllers/au/org/ala/collectory/ProviderMapController.groovy
+++ b/grails-app/controllers/au/org/ala/collectory/ProviderMapController.groovy
@@ -34,7 +34,6 @@ class ProviderMapController {
     def create = {
         def providerMapInstance = new ProviderMap()
         providerMapInstance.properties = params
-        println "createFor = ${params.createFor}"
         if (params.createFor) {
             def pg = Collection.findByUid(params.createFor) as Collection
             if (pg) {
diff --git a/grails-app/controllers/au/org/ala/collectory/PublicController.groovy b/grails-app/controllers/au/org/ala/collectory/PublicController.groovy
index 85f7f65b..b34b13b8 100644
--- a/grails-app/controllers/au/org/ala/collectory/PublicController.groovy
+++ b/grails-app/controllers/au/org/ala/collectory/PublicController.groovy
@@ -243,7 +243,6 @@ class PublicController {
         response.setHeader("Cache-Control","no-cache")
         response.addHeader("Cache-Control","no-store")
         def instance = providerGroupService._get(params.id)
-        //println ">>debug map key " + grailsApplication.config.google.maps.v2.key
         if (!instance) {
             log.error "Unable to find entity for id = ${params.id}"
             def error = ["error":"unable to find entity for id = " + params.id]
@@ -251,17 +250,14 @@ class PublicController {
         } else {
             /* get decade breakdown */
             def decadeUrl = grailsApplication.config.biocacheServicesUrl+ "/breakdown/collection/decades/${instance.generatePermalink()}.json";
-            //println decadeUrl
             def conn = new URL(decadeUrl).openConnection()
             conn.setConnectTimeout 1500
             def dataTable = null
             def json
             try {
                 json = conn.content.text
-                //println "Response = " + json
                 def decades = JSON.parse(json)?.decades
                 dataTable = buildDecadeDataTable(decades)
-                //println "dataTable = " + dataTable
             } catch (SocketTimeoutException e) {
                 log.warn "Timed out getting decade breakdown. URL= ${url}."
                 def result = [error:"Timed out getting decade breakdown.", dataTable: null]
@@ -303,7 +299,6 @@ class PublicController {
         def taxonUrl = grailsApplication.config.biocacheServicesUrl + "/breakdown/{entity}/{uid}?max=" + threshold
         taxonUrl = taxonUrl.replaceFirst(/\{uid\}/, params.id ?: '')
         taxonUrl = taxonUrl.replaceFirst(/\{entity\}/, wsEntityForBreakdown(params.id))
-        //println "taxonUrl: " + taxonUrl
 
         def conn = new URL(taxonUrl).openConnection()
         def jsonResponse = null
@@ -312,7 +307,6 @@ class PublicController {
             conn.setConnectTimeout(10000)
             conn.setReadTimeout(50000)
             jsonResponse = conn.content.text
-            //println "Response = " + json
             //sleep delay
             breakdown = JSON.parse(jsonResponse)?.breakdown ?: JSON.parse(jsonResponse)
         } catch (SocketTimeoutException e) {
@@ -361,7 +355,6 @@ class PublicController {
             conn.setConnectTimeout 10000
             conn.setReadTimeout 50000
             json = conn.content.text
-            //println "Response = " + json
             def breakdown = JSON.parse(json)?.breakdown ?: JSON.parse(json)
             if (breakdown && breakdown.toString() != "null") {
                 dataTable = buildTaxonChartDataTable(breakdown,params.rank,params.name)
@@ -803,9 +796,7 @@ class PublicController {
         // must have at least one value to build a query
         if (instCodes || collCodes) {
             def instClause = instCodes ? buildSearchClause("inst", instCodes) : ""
-            //println instClause
             def collClause = collCodes ? buildSearchClause("coll", collCodes) : ""
-            //println collClause
             def url = grailsApplication.config.biocacheUiURL + "/searchForUID.JSON?pageSize=0" + instClause + collClause
         } else {
             return ""
diff --git a/grails-app/controllers/au/org/ala/collectory/ReportsController.groovy b/grails-app/controllers/au/org/ala/collectory/ReportsController.groovy
index 944e8413..44e00b76 100644
--- a/grails-app/controllers/au/org/ala/collectory/ReportsController.groovy
+++ b/grails-app/controllers/au/org/ala/collectory/ReportsController.groovy
@@ -5,7 +5,7 @@ import grails.converters.JSON
 
 class ReportsController {
 
-    def collectoryAuthService, activityLogService
+    def collectoryAuthService, activityLogService, collectionService, institutionService
 
     def index = {
         redirect action: list, params: params
@@ -86,7 +86,9 @@ class ReportsController {
     }
 
     def codes = {
-        [codeSummaries: (ProviderMap.list().collect { it.collection.buildSummary() }).sort {it.name}]
+        [codeSummaries: (ProviderMap.list().collect {
+            collectionService.buildSummary(it.collection.uid)
+        }).sort {it.name}]
     }
 
     def duplicateContacts = {
@@ -217,13 +219,13 @@ class ReportsController {
     def attributions = {
         def collAttributions = []
         Collection.list([sort: 'name']).each {
-            ProviderGroupSummary pgs = it.buildSummary()
+            ProviderGroupSummary pgs = collectionService.buildSummary(it.uid)
             List<Attribution> attribs = it.getAttributionList()
             def ats = new Attributions(pgs, attribs)
             collAttributions << ats
         }
         def instAttributions = Institution.list([sort: 'name']).collect {
-            ProviderGroupSummary pgs = it.buildSummary()
+            ProviderGroupSummary pgs = institutionService.buildSummary(it.uid)
             List<Attribution> attribs = it.getAttributionList()
             new Attributions(pgs, attribs)
         }
@@ -248,7 +250,7 @@ class ReportsController {
                 }
                 // compare to num digistised
                 if (count == 0 || (count / it.numRecordsDigitised) < 0.7) {
-                    mrs << [collection:it.buildSummary(), biocacheCount: count, claimed: it.numRecordsDigitised]
+                    mrs << [collection: collectionService.buildSummary(it.uid), biocacheCount: count, claimed: it.numRecordsDigitised]
                 }
             }
         }
diff --git a/grails-app/controllers/au/org/ala/collectory/TempDataInterceptor.groovy b/grails-app/controllers/au/org/ala/collectory/TempDataInterceptor.groovy
index 6d8be60a..0e704350 100644
--- a/grails-app/controllers/au/org/ala/collectory/TempDataInterceptor.groovy
+++ b/grails-app/controllers/au/org/ala/collectory/TempDataInterceptor.groovy
@@ -47,7 +47,6 @@ class TempDataInterceptor {
             params.json = request.JSON
             return true
         } catch (Exception e) {
-            println "exception caught ${e}"
             if (request.getContentLength() > 0) {
                 badRequest 'cannot parse request body as JSON'
                 return false
diff --git a/grails-app/domain/au/org/ala/collectory/Collection.groovy b/grails-app/domain/au/org/ala/collectory/Collection.groovy
index d42e8b66..ef4ca6e6 100644
--- a/grails-app/domain/au/org/ala/collectory/Collection.groovy
+++ b/grails-app/domain/au/org/ala/collectory/Collection.groovy
@@ -23,7 +23,7 @@ class Collection implements ProviderGroup, Serializable {
     static final String ENTITY_PREFIX = 'co'
 
     static auditable = [ignore: ['version','dateCreated','lastUpdated','userLastModified']]
-    
+
     String collectionType       // list of type of collection as JSON e.g ['live', 'preserved', 'tissue', 'DNA']
     String active               // tdwg developmentStatus
     int numRecords
@@ -270,58 +270,6 @@ class Collection implements ProviderGroup, Serializable {
         return false
     }
 
-    /**
-     * Returns a list of all hubs this collection belongs to.
-     *
-     * @return list of DataHub
-     */
-    List listHubMembership() {
-        DataHub.list().findAll {it.isCollectionMember(uid)}
-    }
-
-    /**
-     * Returns a summary of the collection including:
-     * - id
-     * - name
-     * - acronym
-     * - lsid if available
-     * - institution (id,uid, name & logo url) if available
-     * - description
-     * - provider codes for matching with biocache records
-     *
-     * @return CollectionSummary
-     */
-    CollectionSummary buildSummary() {
-        CollectionSummary cs = init(new CollectionSummary()) as CollectionSummary
-        if (institution) {
-            cs.institutionName = institution.name
-            cs.institutionId = institution.id
-            cs.institutionUid = institution.uid
-            if (institution.logoRef?.file) {
-                cs.institutionLogoUrl = au.org.ala.collectory.Utilities.buildInstitutionLogoUrl(institution.logoRef.file)
-            }
-        }
-
-        cs.collectionId = cs.id
-        cs.collectionUid = cs.uid
-        cs.collectionName = cs.name
-
-        cs.derivedInstCodes = getListOfInstitutionCodesForLookup()
-        cs.derivedCollCodes = getListOfCollectionCodesForLookup()
-        cs.hubMembership = listHubMembership().collect { [uid: it.uid, name: it.name] }
-        listProviders().each {
-            def pg = Collection.findByUid(it)
-            if (pg) {
-                if (it[0..1] == 'dp') {
-                    cs.relatedDataProviders << [uid: pg.uid, name: pg.name]
-                } else {
-                    cs.relatedDataResources << [uid: pg.uid, name: pg.name]
-                }
-            }
-        }
-        return cs
-    }
-
     /**
      * Returns true if:
      *  a) parent institution is a partner
diff --git a/grails-app/domain/au/org/ala/collectory/DataResource.groovy b/grails-app/domain/au/org/ala/collectory/DataResource.groovy
index 6d548b7b..2c4ad3fd 100644
--- a/grails-app/domain/au/org/ala/collectory/DataResource.groovy
+++ b/grails-app/domain/au/org/ala/collectory/DataResource.groovy
@@ -49,6 +49,8 @@ class DataResource implements ProviderGroup, Serializable {
         dataCollectionProtocolDoc type: "text"
         suitableFor type: "text"
         suitableForOtherDetail type: "text"
+        dataProvider fetch: 'join'
+        institution fetch: 'join'
     }
 
     String rights
@@ -176,44 +178,6 @@ class DataResource implements ProviderGroup, Serializable {
         return false;
     }
 
-    /**
-     * Returns a summary of the data provider including:
-     * - id
-     * - name
-     * - acronym
-     * - lsid if available
-     * - description
-     * - data provider name, id and uid
-     *
-     * @return CollectionSummary
-     */
-    DataResourceSummary buildSummary() {
-        DataResourceSummary drs = init(new DataResourceSummary()) as DataResourceSummary
-        drs.dataProvider = dataProvider?.name
-        drs.dataProviderId = dataProvider?.id
-        drs.dataProviderUid = dataProvider?.uid
-        drs.downloadLimit = downloadLimit
-
-        drs.hubMembership = listHubMembership().collect { [uid: it.uid, name: it.name] }
-        def consumers = listConsumers()
-        consumers.each {
-            def pg = DataResource.findByUid(it)
-            if (pg) {
-                if (it[0..1] == 'co') {
-                    drs.relatedCollections << [uid: pg.uid, name: pg.name]
-                } else {
-                    drs.relatedInstitutions << [uid: pg.uid, name: pg.name]
-                }
-            }
-        }
-        // for backward compatibility
-        if (drs.relatedInstitutions) {
-            drs.institution = drs.relatedInstitutions[0].name
-            drs.institutionUid = drs.relatedInstitutions[0].uid
-        }
-        return drs
-    }
-
     Boolean isVerified(){
 
         if(defaultDarwinCoreValues){
@@ -265,15 +229,6 @@ class DataResource implements ProviderGroup, Serializable {
         save(flush:true)
     }
 
-    /**
-     * Returns a list of all hubs this resource belongs to.
-     *
-     * @return list of DataHub
-     */
-    List listHubMembership() {
-        DataHub.list().findAll {it.isDataResourceMember(uid)}
-    }
-
     /**
      * True if this resource uses a CC license.
      * @return
diff --git a/grails-app/domain/au/org/ala/collectory/Institution.groovy b/grails-app/domain/au/org/ala/collectory/Institution.groovy
index 55748fb6..22a4fa5d 100644
--- a/grails-app/domain/au/org/ala/collectory/Institution.groovy
+++ b/grails-app/domain/au/org/ala/collectory/Institution.groovy
@@ -64,37 +64,6 @@ class Institution implements ProviderGroup, Serializable {
         sort: 'name'
     }
 
-    /**
-     * Returns a summary of the institution including:
-     * - id
-     * - name
-     * - acronym
-     * - lsid if available
-     * - description
-     *
-     * @return InstitutionSummary
-     * @.history 2-8-2010 removed inst codes as these are now related only to collections (can be added back with a different mechanism if required)
-     */
-    InstitutionSummary buildSummary() {
-        InstitutionSummary is = init(new InstitutionSummary()) as InstitutionSummary
-        is.institutionId = dbId()
-        is.institutionUid = uid
-        is.institutionName = name
-        is.collections = collections.collect { [it.uid, it.name] }
-        listProviders().each {
-            def pg = Institution.findByUid(it)
-            if (pg) {
-                if (it[0..1] == 'dp') {
-                    is.relatedDataProviders << [uid: pg.uid, name: pg.name]
-                } else {
-                    is.relatedDataResources << [uid: pg.uid, name: pg.name]
-                }
-            }
-        }
-        is.hubMembership = listHubMembership().collect { [uid: it.uid, name: it.name] }
-        return is
-    }
-
     /**
      * Returns true if:
      *  a) has membership of a collection network (hub) (assumed that all hubs are partners)
@@ -147,15 +116,6 @@ class Institution implements ProviderGroup, Serializable {
         return result
     }
 
-    /**
-     * Returns a list of all hubs this collection belongs to.
-     *
-     * @return list of DataHub
-     */
-    List listHubMembership() {
-        DataHub.list().findAll {it.isInstitutionMember(uid)}
-    }
-
     /**
      * List of institutions that include this as a child institution.
      *
diff --git a/grails-app/services/au/org/ala/collectory/ActivityLogService.groovy b/grails-app/services/au/org/ala/collectory/ActivityLogService.groovy
index 145047a1..cfa2ad04 100644
--- a/grails-app/services/au/org/ala/collectory/ActivityLogService.groovy
+++ b/grails-app/services/au/org/ala/collectory/ActivityLogService.groovy
@@ -10,7 +10,6 @@ class ActivityLogService {
     def log(params) {
         def al = new ActivityLog(params)
         al.timestamp = new Date()
-        al.errors.each {println it}
         al.save(flush:true)
     }
 
@@ -23,7 +22,6 @@ class ActivityLogService {
         //def a = Actions.valueOf(Actions.class, action)
         //def actionText = a ? a.toString() : action
         def al = new ActivityLog(timestamp: new Date(), user: user, admin: isAdmin, action: action.toString())
-        al.errors.each {println it}
         al.save(flush:true)
     }
 
@@ -36,9 +34,7 @@ class ActivityLogService {
     def log(String user, boolean isAdmin, Action action, String item) {
         def al = new ActivityLog(timestamp: new Date(), user: user, admin: isAdmin, action: action.toString() + " " + item)
         al.validate()
-        if (al.hasErrors()) {
-            al.errors.each {println it}
-        }
+
         al.save(flush:true)
     }
 
diff --git a/grails-app/services/au/org/ala/collectory/CollectionService.groovy b/grails-app/services/au/org/ala/collectory/CollectionService.groovy
new file mode 100644
index 00000000..dcacf22a
--- /dev/null
+++ b/grails-app/services/au/org/ala/collectory/CollectionService.groovy
@@ -0,0 +1,69 @@
+/*
+ * Copyright (C) 2011 Atlas of Living Australia
+ * All Rights Reserved.
+ *
+ * The contents of this file are subject to the Mozilla Public
+ * License Version 1.1 (the "License"); you may not use this file
+ * except in compliance with the License. You may obtain a copy of
+ * the License at http://www.mozilla.org/MPL/
+ *
+ * Software distributed under the License is distributed on an "AS
+ * IS" basis, WITHOUT WARRANTY OF ANY KIND, either express or
+ * implied. See the License for the specific language governing
+ * rights and limitations under the License.
+ */
+package au.org.ala.collectory
+
+
+import grails.plugin.cache.Cacheable
+
+class CollectionService {
+
+    static transactional = false
+
+    def dataHubService
+
+    /**
+     * Returns a summary of the collection including:
+     * - id
+     * - name
+     * - acronym
+     * - lsid if available
+     * - institution (id,uid, name & logo url) if available
+     * - description
+     * - provider codes for matching with biocache records
+     *
+     * @return CollectionSummary
+     */
+    CollectionSummary buildSummary(Collection collection) {
+        CollectionSummary cs = collection.init(new CollectionSummary()) as CollectionSummary
+        if (collection.institution) {
+            cs.institutionName = collection.institution.name
+            cs.institutionId = collection.institution.id
+            cs.institutionUid = collection.institution.uid
+            if (collection.institution.logoRef?.file) {
+                cs.institutionLogoUrl = au.org.ala.collectory.Utilities.buildInstitutionLogoUrl(collection.institution.logoRef.file)
+            }
+        }
+
+        cs.collectionId = cs.id
+        cs.collectionUid = cs.uid
+        cs.collectionName = cs.name
+
+        cs.derivedInstCodes = collection.getListOfInstitutionCodesForLookup()
+        cs.derivedCollCodes = collection.getListOfCollectionCodesForLookup()
+        cs.hubMembership = dataHubService.listDataHubs().findAll {it.isCollectionMember(collection.uid)}.collect { [uid: it.uid, name: it.name] }
+        collection.listProviders().each {
+            def pg = Collection.findByUid(it)
+            if (pg) {
+                if (it[0..1] == 'dp') {
+                    cs.relatedDataProviders << [uid: pg.uid, name: pg.name]
+                } else {
+                    cs.relatedDataResources << [uid: pg.uid, name: pg.name]
+                }
+            }
+        }
+        return cs
+    }
+
+}
diff --git a/grails-app/services/au/org/ala/collectory/CollectoryAuthService.groovy b/grails-app/services/au/org/ala/collectory/CollectoryAuthService.groovy
index fa378b2f..15972fa3 100644
--- a/grails-app/services/au/org/ala/collectory/CollectoryAuthService.groovy
+++ b/grails-app/services/au/org/ala/collectory/CollectoryAuthService.groovy
@@ -255,10 +255,12 @@ class CollectoryAuthService{
         Boolean authorised = false
         if(grailsApplication.config.security.apikey.checkEnabled.toBoolean() || grailsApplication.config.security.apikey.enabled.toBoolean()){
             def apiKey = getApiKey(params, request)
-            Call<CheckApiKeyResult> checkApiKeyCall = apiKeyClient.checkApiKey(apiKey)
-            final Response<CheckApiKeyResult> checkApiKeyResponse = checkApiKeyCall.execute()
-            CheckApiKeyResult apiKeyCheck = checkApiKeyResponse.body();
-            authorised = apiKeyCheck.isValid()
+            if (apiKey) {
+                Call<CheckApiKeyResult> checkApiKeyCall = apiKeyClient.checkApiKey(apiKey)
+                final Response<CheckApiKeyResult> checkApiKeyResponse = checkApiKeyCall.execute()
+                CheckApiKeyResult apiKeyCheck = checkApiKeyResponse.body();
+                authorised = apiKeyCheck.isValid()
+            }
         }
 
         if(!authorised){
diff --git a/grails-app/services/au/org/ala/collectory/CrudService.groovy b/grails-app/services/au/org/ala/collectory/CrudService.groovy
index 511b003f..27c631e4 100644
--- a/grails-app/services/au/org/ala/collectory/CrudService.groovy
+++ b/grails-app/services/au/org/ala/collectory/CrudService.groovy
@@ -17,6 +17,7 @@ class CrudService {
     def grailsApplication
     def idGeneratorService
     def providerGroupService
+    def dataHubService
 
     static baseStringProperties = ['guid','name','acronym','phone','email','state','pubShortDescription',
                                    'pubDescription','techDescription','notes', 'isALAPartner','focus','attributions',
@@ -157,7 +158,7 @@ class CrudService {
         }
         return dp
     }
-    
+
     def updateDataProvider(dp, obj) {
         updateBaseProperties(dp, obj)
         updateDataProviderProperties(dp, obj)
@@ -327,7 +328,7 @@ class CrudService {
             }
             use (OutputFormat) {
                 networkMembership = p.networkMembership?.formatNetworkMembership()
-                hubMembership = p.listHubMembership()?.formatHubMembership()
+                hubMembership = dataHubService.listDataHubs()?.findAll {it.isDataResourceMember(p.uid)}?.formatHubMembership()
                 taxonomyCoverageHints = JSONHelper.taxonomyHints(p.taxonomyHints)
                 attributions = p.attributionList.formatAttributions()
                 dateCreated = p.dateCreated
@@ -355,9 +356,9 @@ class CrudService {
                     riskAssessment = p.riskAssessment
                 }
                 contentTypes = p.contentTypes ? p.contentTypes.formatJSON() : []
-                if (p.listConsumers()) {
-                    linkedRecordConsumers = formatEntitiesFromUids(p.listConsumers())
-                }
+//                if (p.listConsumers()) {
+//                    linkedRecordConsumers = formatEntitiesFromUids(p.listConsumers())
+//                }
                 if (p.connectionParameters) {
                     def connParams =  p.connectionParameters.formatJSON()
                     if (!apiKey){
@@ -371,7 +372,7 @@ class CrudService {
                 if (p.defaultDarwinCoreValues) {
                     defaultDarwinCoreValues = p.defaultDarwinCoreValues.formatJSON()
                 }
-                hasMappedCollections = p.hasMappedCollections()
+//                hasMappedCollections = p.hasMappedCollections()
                 status = p.status
                 provenance = p.provenance
                 harvestFrequency = p.harvestFrequency
@@ -386,9 +387,9 @@ class CrudService {
                 isShareableWithGBIF = p.isShareableWithGBIF
                 verified = p.isVerified()
                 gbifRegistryKey = p.gbifRegistryKey
-                if (p.externalIdentifiers) {
-                    externalIdentifiers = p.externalIdentifiers.formatExternalIdentifiers()
-                }
+//                if (p.externalIdentifiers) {
+//                    externalIdentifiers = p.externalIdentifiers.formatExternalIdentifiers()
+//                }
                 doi = p.gbifDoi
             }
         }
@@ -571,7 +572,7 @@ class CrudService {
             }
             use (OutputFormat) {
                 networkMembership = p.networkMembership?.formatNetworkMembership()
-                hubMembership = p.listHubMembership()?.formatHubMembership()
+                hubMembership = dataHubService.listDataHubs()?.findAll {it.isInstitutionMember(p.uid)}?.formatHubMembership()
                 attributions = p.attributionList.formatAttributions()
                 dateCreated = p.dateCreated
                 lastUpdated = p.lastUpdated
@@ -662,7 +663,7 @@ class CrudService {
             }
             use (OutputFormat) {
                 networkMembership = p.networkMembership?.formatNetworkMembership()
-                hubMembership = p.listHubMembership()?.formatHubMembership()
+                hubMembership = dataHubService.listDataHubs()?.findAll {it.isCollectionMember(p.uid)}?.formatHubMembership()
                 taxonomyCoverageHints = JSONHelper.taxonomyHints(p.taxonomyHints)
                 attributions = p.attributionList.formatAttributions()
 
@@ -905,7 +906,6 @@ class CrudService {
         }
         // null objects are copies of JSONObject.NULL - set them to Java null
         [baseStringProperties,dataResourceStringProperties].flatten().each {
-            //println "checking base string property ${it} " + obj.has(it) ? "exists - " + obj."${it}" : "absent"
             if (obj.has(it) && obj."${it}".toString() == 'null') {
                 obj."${it}" = null
             }
@@ -989,7 +989,6 @@ class OutputFormat {
     static def formatAttributions(List list) {
         def result = []
         list.each {
-            //println "attribution ${it.name} - ${it.url}"
             // lookup attribution
             result << [name: it.name, url: it.url]
         }
diff --git a/grails-app/services/au/org/ala/collectory/DataHubService.groovy b/grails-app/services/au/org/ala/collectory/DataHubService.groovy
new file mode 100644
index 00000000..e095a4fa
--- /dev/null
+++ b/grails-app/services/au/org/ala/collectory/DataHubService.groovy
@@ -0,0 +1,27 @@
+/*
+ * Copyright (C) 2011 Atlas of Living Australia
+ * All Rights Reserved.
+ *
+ * The contents of this file are subject to the Mozilla Public
+ * License Version 1.1 (the "License"); you may not use this file
+ * except in compliance with the License. You may obtain a copy of
+ * the License at http://www.mozilla.org/MPL/
+ *
+ * Software distributed under the License is distributed on an "AS
+ * IS" basis, WITHOUT WARRANTY OF ANY KIND, either express or
+ * implied. See the License for the specific language governing
+ * rights and limitations under the License.
+ */
+package au.org.ala.collectory
+
+import grails.plugin.cache.Cacheable
+
+class DataHubService {
+
+    static transactional = false
+
+    @Cacheable('dataHubCache')
+    def listDataHubs() {
+        DataHub.list()
+    }
+}
diff --git a/grails-app/services/au/org/ala/collectory/DataLoaderService.groovy b/grails-app/services/au/org/ala/collectory/DataLoaderService.groovy
index e0cd433e..e88b8d5e 100644
--- a/grails-app/services/au/org/ala/collectory/DataLoaderService.groovy
+++ b/grails-app/services/au/org/ala/collectory/DataLoaderService.groovy
@@ -44,10 +44,9 @@ class DataLoaderService {
              */
             def params = [:]
             dataProvidercolumns.eachWithIndex {it, i ->
-                //println "i=${i} value=${nextLine[i]}"
                 if (nextLine[i]) {
                     // eliminate any \ used for line breaks
-                    params[it] = nextLine[i] //.replaceAll("" + (92 as char)," ")
+                    params[it] = nextLine[i]
                 }
             }
 
@@ -100,12 +99,7 @@ class DataLoaderService {
         def address = params.address
         params.remove('address')  // don't apply un-parsed address directly to entity
         if (hasValue(address)) {
-            //println "---------------"
-            //println "${params.uid} address = ${address}"
             def addressParams = parseAddress(address)
-            /*addressParams.each {k,v ->
-                println "${k} = ${v}"
-            }*/
             if (dp.address) {
                 // dp.address.properties = addressParams doesn't work - readonly for some reason
                 dp.address.street = addressParams.street
@@ -155,10 +149,9 @@ class DataLoaderService {
              */
             def params = [:]
             dataResourcecolumns.eachWithIndex {it, i ->
-                //println "i=${i} value=${nextLine[i]}"
                 if (nextLine[i]) {
                     // eliminate any \ used for line breaks
-                    params[it] = nextLine[i] //.replaceAll("" + (92 as char)," ")
+                    params[it] = nextLine[i]
                 }
             }
 
@@ -224,30 +217,7 @@ class DataLoaderService {
             contactFors = (contactFors)?:[]
             contactFors.add  it
             contactMap.put(entityId, contactFors)
-            //println "added contact ${it.contact.id} to entity ${entityId}"
         }
-        /*imp.contact.each {
-            Contact ct = new Contact()
-            ct.id = it.id
-            ct.firstName = load(it.firstName)
-            ct.lastName = load(it.lastName)
-            ct.phone = load(it.phone)
-            ct.fax = load(it.fax)
-            ct.title = load(it.title)
-            ct.email = load(it.email)
-            ct.userLastModified = load(it.userLastModified)
-            ct.notes = load(it.notes)
-            ct.mobile = load(it.mobile)
-            // no point in these as Grails overrides them
-            ct.dateCreated = loadDate(it.dateCreated)
-            ct.lastUpdated = loadDate(it.dateLastModified)
-
-            if (ct.hasErrors()) {
-                ct.errors.each {println it}
-            } else {
-                ct.save()
-            }
-        }*/
         // clear institutions
         sql.execute("delete from provider_map_provider_code")
         sql.execute("delete from provider_map")
@@ -739,7 +709,7 @@ class DataLoaderService {
                  * save
                  */
                 ProviderGroup provider
-                
+
                 // check whether it's really an institution
                 if (recogniseInstitution(params.name)) {
                     /* provider */
@@ -802,34 +772,6 @@ class DataLoaderService {
                     provider.numRecords = buildSize(params)
                     provider.userLastModified = "BCI loader"
 
-                    /*/ create or assign infosource only if there is some data
-                    String webServiceUri = params.webServiceUri
-                    String webServiceType = params.webServiceProtocol
-
-                    if (webServiceUri) {
-                        // see if there is an existing infosource that matches the access parameters
-                        InfoSource is = InfoSource.list().find {
-                            it.getWebServiceUri() == webServiceUri &&
-                            it.getWebServiceProtocol() == webServiceType
-                        }
-                        if (!is) {
-                            // create a new one
-                            is = new InfoSource(title: "created for " + provider.name)
-                            is.setWebServiceUri webServiceUri
-                            is.setWebServiceProtocol webServiceType
-                        }
-                        provider.infoSource = is
-                        is.addToCollections(provider)
-                        is.userLastModified = "BCI loader"
-
-                        is.save()
-                        if (!is.validate()) {
-                            is.errors.each {
-                                println it
-                            }
-                        }
-                    }*/
-
                 }
 
                 log.info ">> Loading ${provider?.name} as ${provider.groupType}"
@@ -1133,8 +1075,6 @@ class DataLoaderService {
         def params = [:]
         // parse address  eg Plant Pathology Branch, DPIand Fisheries, 80 meiers Rd, Indooroopilly, QLD 4069
         address = trimTrailing(address)
-        //println "---------------"
-        //println "full address = " + address
 
         /* remove possible country at end */
         ['australia','new zealand'].each { country ->
@@ -1145,7 +1085,6 @@ class DataLoaderService {
 
         /* POSTCODE */
         def postcode = address[address.length() - 4 .. address.length() - 1]
-        //println "possible postcode = " + postcode
         try {
             Integer.parseInt(postcode)
             address = address[0 .. address.length() - 5]
@@ -1176,9 +1115,6 @@ class DataLoaderService {
         address = trimTrailing(address)
 
         def bits = address.tokenize(",")
-        /*println "<<bits"
-        bits.each {println it.trim()}
-        println "bits>>"*/
 
         if (bits.size > 1) {
             // if remainder contains a comma then assume the right-most bit is the city
@@ -1211,9 +1147,6 @@ class DataLoaderService {
         if (postcode) {params.postcode = postcode}
         if (postBox) {params.postBox = postBox}
 
-        /*params.each {k,v ->
-            println "${k} = ${v}"
-        }*/
 
         return params
     }
diff --git a/grails-app/services/au/org/ala/collectory/DataResourceService.groovy b/grails-app/services/au/org/ala/collectory/DataResourceService.groovy
new file mode 100644
index 00000000..04dcb2e0
--- /dev/null
+++ b/grails-app/services/au/org/ala/collectory/DataResourceService.groovy
@@ -0,0 +1,63 @@
+/*
+ * Copyright (C) 2011 Atlas of Living Australia
+ * All Rights Reserved.
+ *
+ * The contents of this file are subject to the Mozilla Public
+ * License Version 1.1 (the "License"); you may not use this file
+ * except in compliance with the License. You may obtain a copy of
+ * the License at http://www.mozilla.org/MPL/
+ *
+ * Software distributed under the License is distributed on an "AS
+ * IS" basis, WITHOUT WARRANTY OF ANY KIND, either express or
+ * implied. See the License for the specific language governing
+ * rights and limitations under the License.
+ */
+package au.org.ala.collectory
+
+
+import grails.plugin.cache.Cacheable
+
+class DataResourceService {
+
+    static transactional = false
+
+    def dataHubService
+
+    /**
+     * Returns a summary of the data provider including:
+     * - id
+     * - name
+     * - acronym
+     * - lsid if available
+     * - description
+     * - data provider name, id and uid
+     *
+     * @return CollectionSummary
+     */
+    DataResourceSummary buildSummary(DataResource dataResource) {
+        DataResourceSummary drs = dataResource.init(new DataResourceSummary()) as DataResourceSummary
+        drs.dataProvider = dataResource.dataProvider?.name
+        drs.dataProviderId = dataResource.dataProvider?.id
+        drs.dataProviderUid = dataResource.dataProvider?.uid
+        drs.downloadLimit = dataResource.downloadLimit
+
+        drs.hubMembership = dataHubService.listDataHubs().findAll {it.isDataResourceMember(dataResource.uid)}.collect { [uid: it.uid, name: it.name] }
+        def consumers = dataResource.listConsumers()
+        consumers.each {
+            def pg = DataResource.findByUid(it)
+            if (pg) {
+                if (it[0..1] == 'co') {
+                    drs.relatedCollections << [uid: pg.uid, name: pg.name]
+                } else {
+                    drs.relatedInstitutions << [uid: pg.uid, name: pg.name]
+                }
+            }
+        }
+        // for backward compatibility
+        if (drs.relatedInstitutions) {
+            drs.institution = drs.relatedInstitutions[0].name
+            drs.institutionUid = drs.relatedInstitutions[0].uid
+        }
+        return drs
+    }
+}
diff --git a/grails-app/services/au/org/ala/collectory/InstitutionService.groovy b/grails-app/services/au/org/ala/collectory/InstitutionService.groovy
new file mode 100644
index 00000000..51aeaf41
--- /dev/null
+++ b/grails-app/services/au/org/ala/collectory/InstitutionService.groovy
@@ -0,0 +1,57 @@
+/*
+ * Copyright (C) 2011 Atlas of Living Australia
+ * All Rights Reserved.
+ *
+ * The contents of this file are subject to the Mozilla Public
+ * License Version 1.1 (the "License"); you may not use this file
+ * except in compliance with the License. You may obtain a copy of
+ * the License at http://www.mozilla.org/MPL/
+ *
+ * Software distributed under the License is distributed on an "AS
+ * IS" basis, WITHOUT WARRANTY OF ANY KIND, either express or
+ * implied. See the License for the specific language governing
+ * rights and limitations under the License.
+ */
+package au.org.ala.collectory
+
+
+import grails.plugin.cache.Cacheable
+
+class InstitutionService {
+
+    static transactional = false
+
+    def dataHubService
+
+    /**
+     * Returns a summary of the institution including:
+     * - id
+     * - name
+     * - acronym
+     * - lsid if available
+     * - description
+     *
+     * @return InstitutionSummary
+     * @.history 2-8-2010 removed inst codes as these are now related only to collections (can be added back with a different mechanism if required)
+     */
+    InstitutionSummary buildSummary(Institution institution) {
+        InstitutionSummary is = institution.init(new InstitutionSummary()) as InstitutionSummary
+        is.institutionId = institution.dbId()
+        is.institutionUid = institution.uid
+        is.institutionName = institution.name
+        is.collections = institution.collections.collect { [it.uid, it.name] }
+        institution.listProviders().each {
+            def pg = Institution.findByUid(it)
+            if (pg) {
+                if (it[0..1] == 'dp') {
+                    is.relatedDataProviders << [uid: pg.uid, name: pg.name]
+                } else {
+                    is.relatedDataResources << [uid: pg.uid, name: pg.name]
+                }
+            }
+        }
+        is.hubMembership = dataHubService.listDataHubs().findAll {it.isInstitutionMember(institution.uid)}.collect { [uid: it.uid, name: it.name] }
+        return is
+    }
+
+}
diff --git a/grails-app/services/au/org/ala/collectory/MetadataService.groovy b/grails-app/services/au/org/ala/collectory/MetadataService.groovy
index f1e2fcc9..1e8b4366 100644
--- a/grails-app/services/au/org/ala/collectory/MetadataService.groovy
+++ b/grails-app/services/au/org/ala/collectory/MetadataService.groovy
@@ -15,6 +15,7 @@
 package au.org.ala.collectory
 
 import grails.converters.JSON
+import grails.plugin.cache.Cacheable
 
 class MetadataService {
 
diff --git a/grails-app/services/au/org/ala/collectory/ProviderGroupService.groovy b/grails-app/services/au/org/ala/collectory/ProviderGroupService.groovy
index 9e283e0d..f5e1a878 100644
--- a/grails-app/services/au/org/ala/collectory/ProviderGroupService.groovy
+++ b/grails-app/services/au/org/ala/collectory/ProviderGroupService.groovy
@@ -255,7 +255,6 @@ class ProviderGroupService {
             def th = pg.taxonomyHints ? JSON.parse(pg.taxonomyHints) : [:]
             th.range = rangeList
             pg.taxonomyHints = th as JSON
-            println pg.taxonomyHints
 
             pg.userLastModified = collectoryAuthService?.username()
             if (!pg.hasErrors() && pg.save(flush: true)) {
diff --git a/grails-app/taglib/au/org/ala/collectory/CollectoryTagLib.groovy b/grails-app/taglib/au/org/ala/collectory/CollectoryTagLib.groovy
index e93267bf..75536461 100644
--- a/grails-app/taglib/au/org/ala/collectory/CollectoryTagLib.groovy
+++ b/grails-app/taglib/au/org/ala/collectory/CollectoryTagLib.groovy
@@ -686,7 +686,7 @@ class CollectoryTagLib {
         if (!email)
             email = body().toString()
         int index = email.indexOf('@')
-        //println "index=${index}"
+
         if (index > 0) {
             email = email.replaceAll("@", strEncodedAtSign)
         }
@@ -1084,8 +1084,6 @@ class CollectoryTagLib {
      * Draw elements for taxa breakdown chart
      */
     def taxonChart = { attrs ->
-        //println "taxonChart records link"
-        //println buildRecordsUrl(attrs.uid)
         out <<
                 "<div id='taxonRecordsLink' style='visibility:hidden;'>" +
                 " <span id='viewRecordsLink' class='taxonChartCaption'><a class='recordsLink' href='${buildRecordsUrl(attrs.uid)}'>View all records</a></span><br/>" +
@@ -1272,7 +1270,6 @@ class CollectoryTagLib {
      */
     def cleanString = { attrs ->
         def text = attrs.value
-        println text
         // detect json array of strings
         if (text && text.size() > 1 && text[0..1] == '["' && text[text.size()-2..text.size()-1] == '"]') {
             def list = JSON.parse(text.toString())
@@ -1560,7 +1557,6 @@ class CollectoryTagLib {
 
     def showRecordsExceptions = {attrs ->
         def exceptions = attrs.exceptions
-        println exceptions
         if (exceptions) {
             out << '<div class="child-institutions">'
             switch (exceptions.listType) {
@@ -2077,7 +2073,6 @@ class CollectoryTagLib {
 
     def toJson = { attrs ->
         def json = attrs.obj as JSON
-        println json
         out << json.toString()
     }
 
diff --git a/src/main/groovy/au/org/ala/collectory/Classification.groovy b/src/main/groovy/au/org/ala/collectory/Classification.groovy
index 9f34d635..e18838d2 100644
--- a/src/main/groovy/au/org/ala/collectory/Classification.groovy
+++ b/src/main/groovy/au/org/ala/collectory/Classification.groovy
@@ -28,7 +28,6 @@ class Classification {
 
         def filters = filterString.tokenize(",")
         for (filter in filters) {
-            //println "Checking filter ${filter} against keywords ${keywords}"
             if (keywords =~ filter) {
                 return true;
             } else {
diff --git a/src/main/groovy/au/org/ala/collectory/GBIFLoadSummary.groovy b/src/main/groovy/au/org/ala/collectory/GBIFLoadSummary.groovy
index ae132062..324fe9b1 100644
--- a/src/main/groovy/au/org/ala/collectory/GBIFLoadSummary.groovy
+++ b/src/main/groovy/au/org/ala/collectory/GBIFLoadSummary.groovy
@@ -34,9 +34,7 @@ class GBIFLoadSummary {
     //List completedLoads=[]
     List loads = []
     def isLoadRunning(){
-        //System.out.println("Testing to see if it is still running")
         GBIFActiveLoad firstItemNotFinished =  loads.find{!it.isComplete()}
-        //System.out.println("First item : " + firstItemNotFinished)
         return firstItemNotFinished != null
     }
     def getPercentageComplete(){
diff --git a/src/main/groovy/au/org/ala/collectory/ProviderGroup.groovy b/src/main/groovy/au/org/ala/collectory/ProviderGroup.groovy
index 2e00ba2d..37d3ff19 100644
--- a/src/main/groovy/au/org/ala/collectory/ProviderGroup.groovy
+++ b/src/main/groovy/au/org/ala/collectory/ProviderGroup.groovy
@@ -100,9 +100,6 @@ trait ProviderGroup implements Serializable {
         ContactFor.withTransaction {
             cf.save(flush: true)
         }
-        if (cf.hasErrors()) {
-            cf.errors.each {println it.toString()}
-        }
         cf
     }
 
@@ -480,14 +477,6 @@ trait ProviderGroup implements Serializable {
         return pgs
     }
 
-
-
-    /**
-     * Returns a summary object that extends ProviderGroupSummary and is specific to the type of entity.
-     * @return summary object
-     */
-    abstract ProviderGroupSummary buildSummary()
-
     /**
      * Returns a list of UIDs of data providers and data resources that contribute records to the entity.
      * @return

From 461742d48f8970a9038afae642bbac63058c39a5 Mon Sep 17 00:00:00 2001
From: Adam Collins <adam.collins@csiro.au>
Date: Wed, 22 Nov 2023 12:03:26 +1000
Subject: [PATCH 04/13] create relationships for ExternalIdentifier

---
 grails-app/conf/logback.old.groovy            | 78 -------------------
 .../collectory/ProviderGroupController.groovy | 33 ++++----
 .../au/org/ala/collectory/Collection.groovy   |  2 +
 .../au/org/ala/collectory/DataProvider.groovy |  2 +-
 .../au/org/ala/collectory/DataResource.groovy |  3 +
 .../au/org/ala/collectory/Institution.groovy  |  2 +-
 grails-app/migrations/5.1.0.sql               | 41 ++++++++++
 grails-app/migrations/changelog.xml           |  4 +
 .../au/org/ala/collectory/CrudService.groovy  | 14 ++--
 .../ala/collectory/ExternalDataService.groovy |  5 +-
 .../ExternalIdentifierService.groovy          | 60 ++++++++++++++
 .../org/ala/collectory/ProviderGroup.groovy   | 32 --------
 12 files changed, 140 insertions(+), 136 deletions(-)
 delete mode 100644 grails-app/conf/logback.old.groovy
 create mode 100644 grails-app/migrations/5.1.0.sql
 create mode 100644 grails-app/services/au/org/ala/collectory/ExternalIdentifierService.groovy

diff --git a/grails-app/conf/logback.old.groovy b/grails-app/conf/logback.old.groovy
deleted file mode 100644
index 4e988470..00000000
--- a/grails-app/conf/logback.old.groovy
+++ /dev/null
@@ -1,78 +0,0 @@
-import ch.qos.logback.core.util.FileSize
-import org.springframework.boot.logging.logback.ColorConverter
-import org.springframework.boot.logging.logback.WhitespaceThrowableProxyConverter
-
-import java.nio.charset.StandardCharsets
-
-conversionRule 'clr', ColorConverter
-conversionRule 'wex', WhitespaceThrowableProxyConverter
-
-logDir = "/tmp/"
-
-if (new File("/var/log/tomcat9/").exists()){
-    logDir = "/var/log/tomcat9/"
-}
-
-// See http://logback.qos.ch/manual/groovy.html for details on configuration
-appender('STDOUT', ConsoleAppender) {
-    encoder(PatternLayoutEncoder) {
-        charset = StandardCharsets.UTF_8
-        pattern =
-                '%clr(%d{yyyy-MM-dd HH:mm:ss.SSS}){faint} ' + // Date
-                '%clr(%5p) ' + // Log level
-                '%clr(---){faint} %clr([%15.15t]){faint} ' + // Thread
-                '%clr(%-40.40logger{39}){cyan} %clr(:){faint} ' + // Logger
-                '%m%n%wex' // Message
-    }
-}
-
-appender('COLLECTORY_LOG', RollingFileAppender) {
-    file = logDir + "collectory.log"
-    encoder(PatternLayoutEncoder) {
-        pattern =
-                '%d{yyyy-MM-dd HH:mm:ss.SSS} ' + // Date
-                        '%5p ' + // Log level
-                        '--- [%15.15t] ' + // Thread
-                        '%-40.40logger{39} : ' + // Logger
-                        '%m%n%wex' // Message
-    }
-    rollingPolicy(FixedWindowRollingPolicy) {
-        fileNamePattern = logDir + "collectory.log.%i.log.gz"
-        minIndex=1
-        maxIndex=4
-    }
-    triggeringPolicy(SizeBasedTriggeringPolicy) {
-        maxFileSize = FileSize.valueOf('10MB')
-    }
-}
-
-root(INFO, ['STDOUT'])
-
-
-final error = [
-        'org.springframework',
-        'grails.app',
-        'grails.plugins.mail',
-        'org.hibernate',
-        'org.quartz',
-        'asset.pipeline'
-]
-final warn = [
-        'au.org.ala.cas'
-
-]
-final info = [
-        'au.org.ala.collectory',
-        'org.liquibase',
-        'liquibase'
-]
-
-final debug = []
-final trace = []
-
-for (def name : error) logger(name, ERROR, ['COLLECTORY_LOG', 'STDOUT'], false)
-for (def name : warn) logger(name, WARN, ['COLLECTORY_LOG','STDOUT'], false)
-for (def name: info) logger(name, INFO,['COLLECTORY_LOG','STDOUT'], false)
-for (def name: debug) logger(name, DEBUG,['COLLECTORY_LOG','STDOUT'], false)
-for (def name: trace) logger(name, TRACE,['COLLECTORY_LOG','STDOUT'], false)
-
diff --git a/grails-app/controllers/au/org/ala/collectory/ProviderGroupController.groovy b/grails-app/controllers/au/org/ala/collectory/ProviderGroupController.groovy
index ad93f461..c81bf69a 100644
--- a/grails-app/controllers/au/org/ala/collectory/ProviderGroupController.groovy
+++ b/grails-app/controllers/au/org/ala/collectory/ProviderGroupController.groovy
@@ -20,6 +20,7 @@ abstract class ProviderGroupController {
     int TRUNCATE_LENGTH = 255
 
     def idGeneratorService, collectoryAuthService, metadataService, gbifService, dataImportService, providerGroupService, activityLogService
+    def externalIdentifierService
 
     /*
      * Access control
@@ -407,28 +408,30 @@ abstract class ProviderGroupController {
             def sources = params.findAll { key, value ->
                 key.startsWith('source_') && value
             }
-            def external = sources.sort().collect { key, value ->
+
+            // remove existing external identifiers
+            if (pg.externalIdentifiers) {
+                def existing = pg.externalIdentifiers
+                pg.externalIdentifiers = []
+
+                DataResource.withTransaction {
+                    pg.save(flush: true)
+                }
+
+                existing.each { ext -> DataResource.withTransaction { ext.delete(flush: true) } }
+            }
+
+            // add new external identifiers
+            sources.sort().collect { key, value ->
                 def idx = key.substring(7)
                 def source = params[key]
                 def identifier = params."identifier_${idx}"
                 def uri = params."uri_${idx}"
                 if (!uri)
                     uri = null
-                return new ExternalIdentifier(entityUid: pg.uid, source: source, identifier: identifier, uri: uri)
-            }
-            def existing = pg.externalIdentifiers
-            external.each { ext ->
-                def old = existing.find { prev -> prev.same(ext) }
-                if (!old) {
-                    DataResource.withTransaction { ext.save(flush: true) }
-                } else {
-                    old.uri = ext.uri
-                    DataResource.withTransaction { old.save(flush: true) }
-                    existing.remove(old)
-                }
+                externalIdentifierService.addExternalIdentifier(pg.uid, identifier, source, uri)
             }
-            // Delete non-matching, existing external IDs
-            existing.each { ext -> DataResource.withTransaction { ext.delete(flush: true) } }
+
             pg.userLastModified = collectoryAuthService?.username()
             if (!pg.hasErrors()) {
                 DataResource.withTransaction {
diff --git a/grails-app/domain/au/org/ala/collectory/Collection.groovy b/grails-app/domain/au/org/ala/collectory/Collection.groovy
index ef4ca6e6..c4bef4c7 100644
--- a/grails-app/domain/au/org/ala/collectory/Collection.groovy
+++ b/grails-app/domain/au/org/ala/collectory/Collection.groovy
@@ -66,6 +66,8 @@ class Collection implements ProviderGroup, Serializable {
     // maps to exactly one providerMap
     static hasOne = [providerMap: ProviderMap]
 
+    static hasMany = [externalIdentifiers: ExternalIdentifier]
+
     static transients =  ['listOfCollectionCodesForLookup', 'listOfinstitutionCodesForLookup','mappable','inexactlyMapped','attributionList']
 
     static mapping = {
diff --git a/grails-app/domain/au/org/ala/collectory/DataProvider.groovy b/grails-app/domain/au/org/ala/collectory/DataProvider.groovy
index b4ee74aa..9dbc42ef 100644
--- a/grails-app/domain/au/org/ala/collectory/DataProvider.groovy
+++ b/grails-app/domain/au/org/ala/collectory/DataProvider.groovy
@@ -7,7 +7,7 @@ class DataProvider implements ProviderGroup, Serializable {
 
     static auditable = [ignore: ['version','dateCreated','lastUpdated','userLastModified']]
 
-    static hasMany = [resources: DataResource]
+    static hasMany = [resources: DataResource, externalIdentifiers: ExternalIdentifier]
 
     String hiddenJSON // web service only (non-UI) JSON; used by fieldcapture to store project data
 
diff --git a/grails-app/domain/au/org/ala/collectory/DataResource.groovy b/grails-app/domain/au/org/ala/collectory/DataResource.groovy
index 2c4ad3fd..6279ff0b 100644
--- a/grails-app/domain/au/org/ala/collectory/DataResource.groovy
+++ b/grails-app/domain/au/org/ala/collectory/DataResource.groovy
@@ -51,6 +51,7 @@ class DataResource implements ProviderGroup, Serializable {
         suitableForOtherDetail type: "text"
         dataProvider fetch: 'join'
         institution fetch: 'join'
+        externalIdentifiers fetch: 'join'
     }
 
     String rights
@@ -103,6 +104,8 @@ class DataResource implements ProviderGroup, Serializable {
 
     String gbifDoi
 
+    static hasMany = [externalIdentifiers: ExternalIdentifier]
+
     static constraints = {
         guid(nullable:true, maxSize:256)
         uid(blank:false, maxSize:20)
diff --git a/grails-app/domain/au/org/ala/collectory/Institution.groovy b/grails-app/domain/au/org/ala/collectory/Institution.groovy
index 22a4fa5d..6073b449 100644
--- a/grails-app/domain/au/org/ala/collectory/Institution.groovy
+++ b/grails-app/domain/au/org/ala/collectory/Institution.groovy
@@ -14,7 +14,7 @@ class Institution implements ProviderGroup, Serializable {
     String gbifCountryToAttribute      // the 3 digit iso code of the country to attribute in GBIF
 
     // an institution may have many collections
-    static hasMany = [collections: Collection]
+    static hasMany = [collections: Collection, externalIdentifiers: ExternalIdentifier]
 
     static constraints = {
         guid(nullable:true, maxSize:256)
diff --git a/grails-app/migrations/5.1.0.sql b/grails-app/migrations/5.1.0.sql
new file mode 100644
index 00000000..541eadf9
--- /dev/null
+++ b/grails-app/migrations/5.1.0.sql
@@ -0,0 +1,41 @@
+CREATE TABLE `collection_external_identifier` (
+    `collection_external_identifiers_id` bigint NOT NULL,
+    `external_identifier_id` bigint DEFAULT NULL,
+    KEY `FK8k1v2k7qw2k3dyqnhf3xg1i86` (`external_identifier_id`),
+    CONSTRAINT `FK8k1v2k7qw2k3dyqnhf3xg1i86` FOREIGN KEY (`external_identifier_id`) REFERENCES `external_identifier` (`id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8;
+
+insert into `collection_external_identifier` (`collection_external_identifiers_id`, `external_identifier_id`) select c.id, e.id from external_identifier e inner join collection c on e.entity_uid = c.uid;
+
+CREATE TABLE `data_provider_external_identifier` (
+    `data_provider_external_identifiers_id` bigint NOT NULL,
+    `external_identifier_id` bigint DEFAULT NULL,
+    KEY `FKjxuqgndu3vxuvi9rn4vo93e36` (`external_identifier_id`),
+    KEY `FKd92u2i4h2j9i4pqq3dppoatf` (`data_provider_external_identifiers_id`),
+    CONSTRAINT `FKd92u2i4h2j9i4pqq3dppoatf` FOREIGN KEY (`data_provider_external_identifiers_id`) REFERENCES `data_provider` (`id`),
+    CONSTRAINT `FKjxuqgndu3vxuvi9rn4vo93e36` FOREIGN KEY (`external_identifier_id`) REFERENCES `external_identifier` (`id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8;
+
+insert into `data_provider_external_identifier` (`data_provider_external_identifiers_id`, `external_identifier_id`) select c.id, e.id from external_identifier e inner join data_provider c on e.entity_uid = c.uid;
+
+CREATE TABLE `data_resource_external_identifier` (
+    `data_resource_external_identifiers_id` bigint NOT NULL,
+    `external_identifier_id` bigint DEFAULT NULL,
+    KEY `FKjl7sq5yrmiprjejwno3qhosn9` (`external_identifier_id`),
+    KEY `FKhoiygwx6wruolguio186lflgw` (`data_resource_external_identifiers_id`),
+    CONSTRAINT `FKhoiygwx6wruolguio186lflgw` FOREIGN KEY (`data_resource_external_identifiers_id`) REFERENCES `data_resource` (`id`),
+    CONSTRAINT `FKjl7sq5yrmiprjejwno3qhosn9` FOREIGN KEY (`external_identifier_id`) REFERENCES `external_identifier` (`id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8;
+
+insert into `data_resource_external_identifier` (`data_resource_external_identifiers_id`, `external_identifier_id`) select c.id, e.id from external_identifier e inner join data_resource c on e.entity_uid = c.uid;
+
+CREATE TABLE `institution_external_identifier` (
+    `institution_external_identifiers_id` bigint NOT NULL,
+    `external_identifier_id` bigint DEFAULT NULL,
+    KEY `FK1msq9xx132sf5gev64x1yi0ps` (`external_identifier_id`),
+    KEY `FKmqbpx6oa6dnybgvaqhp4bbaqd` (`institution_external_identifiers_id`),
+    CONSTRAINT `FK1msq9xx132sf5gev64x1yi0ps` FOREIGN KEY (`external_identifier_id`) REFERENCES `external_identifier` (`id`),
+    CONSTRAINT `FKmqbpx6oa6dnybgvaqhp4bbaqd` FOREIGN KEY (`institution_external_identifiers_id`) REFERENCES `institution` (`id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8;
+
+insert into `institution_external_identifier` (`institution_external_identifiers_id`, `external_identifier_id`) select c.id, e.id from external_identifier e inner join institution c on e.entity_uid = c.uid;
diff --git a/grails-app/migrations/changelog.xml b/grails-app/migrations/changelog.xml
index 0ab6370f..f219dba3 100644
--- a/grails-app/migrations/changelog.xml
+++ b/grails-app/migrations/changelog.xml
@@ -432,4 +432,8 @@ liquibase.command.referencePassword: XXXXX
         </sql>
     </changeSet>
 
+    <changeSet author="ALA Dev Team" id="20231122-00">
+        <sqlFile path="5.1.0.sql" relativeToChangelogFile="true" encoding="ISO-8859-1" stripComments="1" />
+    </changeSet>
+
 </databaseChangeLog>
diff --git a/grails-app/services/au/org/ala/collectory/CrudService.groovy b/grails-app/services/au/org/ala/collectory/CrudService.groovy
index 27c631e4..3cf0f90f 100644
--- a/grails-app/services/au/org/ala/collectory/CrudService.groovy
+++ b/grails-app/services/au/org/ala/collectory/CrudService.groovy
@@ -356,9 +356,9 @@ class CrudService {
                     riskAssessment = p.riskAssessment
                 }
                 contentTypes = p.contentTypes ? p.contentTypes.formatJSON() : []
-//                if (p.listConsumers()) {
-//                    linkedRecordConsumers = formatEntitiesFromUids(p.listConsumers())
-//                }
+                if (p.listConsumers()) {
+                    linkedRecordConsumers = formatEntitiesFromUids(p.listConsumers())
+                }
                 if (p.connectionParameters) {
                     def connParams =  p.connectionParameters.formatJSON()
                     if (!apiKey){
@@ -372,7 +372,7 @@ class CrudService {
                 if (p.defaultDarwinCoreValues) {
                     defaultDarwinCoreValues = p.defaultDarwinCoreValues.formatJSON()
                 }
-//                hasMappedCollections = p.hasMappedCollections()
+                hasMappedCollections = p.hasMappedCollections()
                 status = p.status
                 provenance = p.provenance
                 harvestFrequency = p.harvestFrequency
@@ -387,9 +387,9 @@ class CrudService {
                 isShareableWithGBIF = p.isShareableWithGBIF
                 verified = p.isVerified()
                 gbifRegistryKey = p.gbifRegistryKey
-//                if (p.externalIdentifiers) {
-//                    externalIdentifiers = p.externalIdentifiers.formatExternalIdentifiers()
-//                }
+                if (p.externalIdentifiers) {
+                    externalIdentifiers = p.externalIdentifiers.formatExternalIdentifiers()
+                }
                 doi = p.gbifDoi
             }
         }
diff --git a/grails-app/services/au/org/ala/collectory/ExternalDataService.groovy b/grails-app/services/au/org/ala/collectory/ExternalDataService.groovy
index 358f4b81..3cfd72a3 100644
--- a/grails-app/services/au/org/ala/collectory/ExternalDataService.groovy
+++ b/grails-app/services/au/org/ala/collectory/ExternalDataService.groovy
@@ -25,6 +25,7 @@ class ExternalDataService {
 
     def grailsApplication
     def crudService
+    def externalIdentifierService
 
     static final POLL_INTERVAL = 15000
     static final DateFormat ALA_TIMESTAMP_FORMAT= new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss")
@@ -126,11 +127,11 @@ class ExternalDataService {
                 if (!dr) {
                     throw new ExternalResourceException("Can't find resource", "manage.note.note12", resource.uid)
                 }
-                ext = dr.getExternalIdentifiers().find({
+                ext = dr.externalIdentifiers.find({
                     it.source == adaptor.source && it.identifier == resource.guid
                 })
                 if (dr && !ext) {
-                    dr.addExternalIdentifier(resource.guid, adaptor.source, resource.source)
+                    externalIdentifierService.addExternalIdentifier(dr.uid, resource.guid, adaptor.source, resource.source)
                 }
             }
 
diff --git a/grails-app/services/au/org/ala/collectory/ExternalIdentifierService.groovy b/grails-app/services/au/org/ala/collectory/ExternalIdentifierService.groovy
new file mode 100644
index 00000000..db9b94bb
--- /dev/null
+++ b/grails-app/services/au/org/ala/collectory/ExternalIdentifierService.groovy
@@ -0,0 +1,60 @@
+/**
+ * Copyright (C) 2011 Atlas of Living Australia
+ * All Rights Reserved.
+ *
+ * The contents of this file are subject to the Mozilla Public
+ * License Version 1.1 (the "License"); you may not use this file
+ * except in compliance with the License. You may obtain a copy of
+ * the License at http://www.mozilla.org/MPL/
+ *
+ * Software distributed under the License is distributed on an "AS
+ * IS" basis, WITHOUT WARRANTY OF ANY KIND, either express or
+ * implied. See the License for the specific language governing
+ * rights and limitations under the License.
+ */
+package au.org.ala.collectory
+
+import groovy.xml.StreamingMarkupBuilder
+import groovy.xml.XmlUtil
+
+import java.text.DateFormat
+import java.text.SimpleDateFormat
+
+class ExternalIdentifierService {
+
+    static transactional = true
+
+    /**
+     * Add an external identifier to this object
+     *
+     * @param identifier The identifier
+     * @param source The identifier source (eg. 'GBIF')
+     * @param link A link to the oreiginal source
+     * @return
+     */
+    ExternalIdentifier addExternalIdentifier(String uid, String identifier, String source, String link) {
+        ExternalIdentifier ext = new ExternalIdentifier(entityUid: uid, identifier: identifier, source: source, uri: link)
+        ExternalIdentifier.withTransaction {
+            ext.save(flush: true)
+        }
+
+        if (uid[0..1] == 'co') {
+            Collection c = Collection.findByUid(uid)
+            c.externalIdentifiers.add(ext)
+            Collection.withTransaction { c.save(flush: true) }
+        } else if (uid[0..1] == 'in') {
+            Institution c = Institution.findByUid(uid)
+            c.externalIdentifiers.add(ext)
+            Institution.withTransaction { c.save(flush: true) }
+        } else if (uid[0..1] == 'dp') {
+            DataProvider c = DataProvider.findByUid(uid)
+            c.externalIdentifiers.add(ext)
+            DataProvider.withTransaction { c.save(flush: true) }
+        } else if (uid[0..1] == 'dr') {
+            DataResource c = DataResource.findByUid(uid)
+            c.externalIdentifiers.add(ext)
+            DataResource.withTransaction { c.save(flush: true) }
+        }
+        return ext
+    }
+}
diff --git a/src/main/groovy/au/org/ala/collectory/ProviderGroup.groovy b/src/main/groovy/au/org/ala/collectory/ProviderGroup.groovy
index 37d3ff19..05c7c475 100644
--- a/src/main/groovy/au/org/ala/collectory/ProviderGroup.groovy
+++ b/src/main/groovy/au/org/ala/collectory/ProviderGroup.groovy
@@ -227,38 +227,6 @@ trait ProviderGroup implements Serializable {
         ContactFor.findByEntityUidAndContact(uid, contact)?.delete()
     }
 
-    /**
-     * Add an external identifier to this object
-     *
-     * @param identifier The identifier
-     * @param source The identifier source (eg. 'GBIF')
-     * @param link A link to the oreiginal source
-     * @return
-     */
-    ExternalIdentifier addExternalIdentifier(String identifier, String source, String link) {
-        ExternalIdentifier ext = new ExternalIdentifier(entityUid: uid, identifier: identifier, source: source, uri: link)
-        ExternalIdentifier.withTransaction {
-            ext.save(flush: true)
-        }
-//        if (ext.hasErrors()) {
-//            ext.errors.each { log.error(it.toString()) }
-//        }
-        return ext
-    }
-
-    /**
-     * Get the external identifiers associated with this entity
-     *
-     * @return The external identifiers
-     */
-    List<ExternalIdentifier> getExternalIdentifiers() {
-        if (uid != null) {
-            return ExternalIdentifier.findAllByEntityUid(uid)
-        } else {
-            return []
-        }
-    }
-
     /**
      * Remove an external identifier
      *

From e22d6e1613613f84da0ae171b7b65437c3a3b8c8 Mon Sep 17 00:00:00 2001
From: Adam Collins <adam.collins@csiro.au>
Date: Wed, 29 Nov 2023 13:12:59 +1000
Subject: [PATCH 05/13] replace DataLink with joins

---
 grails-app/conf/application.yml               |   1 +
 .../CollectoryWebServicesInterceptor.groovy   |   5 +
 .../org/ala/collectory/DataController.groovy  |  60 ++++++++--
 .../ala/collectory/DataLinkController.groovy  | 108 ------------------
 .../collectory/DataProviderController.groovy  |  42 ++++---
 .../collectory/DataResourceController.groovy  |  44 ++++---
 .../org/ala/collectory/GbifController.groovy  |  17 +--
 .../ala/collectory/LookupController.groovy    |   8 +-
 .../collectory/ProviderGroupController.groovy |  18 ++-
 .../ala/collectory/ReportsController.groovy   |  14 ++-
 .../au/org/ala/collectory/Collection.groovy   |  27 ++++-
 .../au/org/ala/collectory/DataLink.groovy     |  16 ---
 .../au/org/ala/collectory/DataProvider.groovy |  31 ++---
 .../au/org/ala/collectory/DataResource.groovy |  10 +-
 .../au/org/ala/collectory/Institution.groovy  |  34 ++++--
 grails-app/migrations/5.1.0.1.sql             |  44 +++++++
 grails-app/migrations/changelog.xml           |   6 +
 .../ala/collectory/CollectionService.groovy   |  13 +--
 .../collectory/CollectoryAuthService.groovy   |   6 +-
 .../au/org/ala/collectory/CrudService.groovy  |  19 +--
 .../ala/collectory/DataResourceService.groovy |  13 +--
 .../ala/collectory/EmlRenderService.groovy    |  14 ++-
 .../ala/collectory/GbifRegistryService.groovy |  61 +++-------
 .../ala/collectory/InstitutionService.groovy  |  14 +--
 .../collectory/ProviderGroupService.groovy    |  28 ++++-
 .../ala/collectory/CollectoryTagLib.groovy    |  36 +++---
 grails-app/views/dataLink/create.gsp          |  44 -------
 grails-app/views/dataLink/edit.gsp            |  52 ---------
 grails-app/views/dataLink/list.gsp            |  65 -----------
 grails-app/views/dataLink/show.gsp            |  67 -----------
 grails-app/views/shared/_consumers.gsp        |  12 +-
 grails-app/views/shared/_providers.gsp        |  23 ++--
 .../org/ala/collectory/ProviderGroup.groovy   |  24 ++--
 33 files changed, 405 insertions(+), 571 deletions(-)
 delete mode 100644 grails-app/controllers/au/org/ala/collectory/DataLinkController.groovy
 delete mode 100644 grails-app/domain/au/org/ala/collectory/DataLink.groovy
 create mode 100644 grails-app/migrations/5.1.0.1.sql
 delete mode 100644 grails-app/views/dataLink/create.gsp
 delete mode 100644 grails-app/views/dataLink/edit.gsp
 delete mode 100644 grails-app/views/dataLink/list.gsp
 delete mode 100644 grails-app/views/dataLink/show.gsp

diff --git a/grails-app/conf/application.yml b/grails-app/conf/application.yml
index 70f9bc72..6f0f28c3 100644
--- a/grails-app/conf/application.yml
+++ b/grails-app/conf/application.yml
@@ -403,6 +403,7 @@ environments:
             url: jdbc:mysql://localhost:3306/collectory?autoReconnect=true&connectTimeout=0&useUnicode=true&characterEncoding=UTF-8&serverTimezone=Australia/Sydney
             username: specieslist
             password: "specieslist"
+            logSql: false
     test:
         grails:
             assets:
diff --git a/grails-app/controllers/au/org/ala/collectory/CollectoryWebServicesInterceptor.groovy b/grails-app/controllers/au/org/ala/collectory/CollectoryWebServicesInterceptor.groovy
index 736e31b4..10b32900 100644
--- a/grails-app/controllers/au/org/ala/collectory/CollectoryWebServicesInterceptor.groovy
+++ b/grails-app/controllers/au/org/ala/collectory/CollectoryWebServicesInterceptor.groovy
@@ -33,6 +33,11 @@ class CollectoryWebServicesInterceptor {
 
     }
     boolean before() {
+        // Failed to intercept this request in UrlMappings, doing it here and in `saveEntity`
+        if (request.method == "POST" && !params.uid) {
+            return true
+        }
+
         // set default role requirement for protected ROLE_EDITOR as the same info is only available to ROLE_EDITOR via the UI.
         String requiredRole = grailsApplication.config.ROLE_EDITOR
 
diff --git a/grails-app/controllers/au/org/ala/collectory/DataController.groovy b/grails-app/controllers/au/org/ala/collectory/DataController.groovy
index e2c396a5..425e1a57 100644
--- a/grails-app/controllers/au/org/ala/collectory/DataController.groovy
+++ b/grails-app/controllers/au/org/ala/collectory/DataController.groovy
@@ -43,7 +43,7 @@ class DataController {
         def uid = params.uid
         if (uid) {
             // it must exist
-            def pg = uid.startsWith('drt') ? TempDataResource.findByUid(uid) : providerGroupService._get(uid)
+            def pg = uid.startsWith('drt') ? TempDataResource.findByUid(uid) : providerGroupService._getEager(uid)
             if (pg) {
                 params.pg = pg
                 // if entity is specified, the instance must be of type entity
@@ -283,6 +283,12 @@ class DataController {
     @Path("/ws/{entity}/{uid}")
     @Produces("application/json")
     def saveEntity() {
+        // Failed to intercept this request in UrlMappings, doing it here and in `CollectoryWebServicesInterceptor`
+        if (!params.uid) {
+            getEntities()
+            return
+        }
+
         def ok = check(params)
         if (ok) {
             def pg = params.pg
@@ -487,6 +493,48 @@ class DataController {
         }
     }
 
+    @Path("/ws/{entity}")
+    @Produces("application/json")
+    def getEntities() {
+
+        def result = []
+
+        def authCheck = false
+        if (params.entity == 'dataResource') {
+            // this auth check (JWT or API key) is a special case handling to support backwards compatibility(which used to check for API key).
+            String requiredRoles = grailsApplication.config.ROLE_ADMIN
+            authCheck = collectoryAuthService.isAuthorisedWsRequest(getParams(), request, response, requiredRoles, null)
+        }
+
+        def clazz = capitalise(params.entity)
+
+        request.JSON.each {
+            def item
+            if (params.entity == 'tempDataResource') {
+                item = crudService.readTempDataResource(it)
+            } else {
+                def pg = providerGroupService._getEager(it)
+                if (pg) {
+                    if (params.entity == 'dataResource') {
+                        item = crudService."read${clazz}"(pg, authCheck)
+                    } else {
+                        item = crudService."read${clazz}"(pg)
+                    }
+                } else {
+                    log.debug("cannot find: " + it)
+                }
+            }
+            if (item) {
+                result.add(metadataService.convertAnyLocalPaths(item))
+            }
+        }
+
+        response.setContentType("application/json")
+        response.setCharacterEncoding("UTF-8")
+
+        render result as JSON
+    }
+
     @Operation(
             method = "GET",
             tags = "collection, institution, dataProvider, dataResource, tempDataResource, dataHub",
@@ -799,7 +847,7 @@ class DataController {
     @Produces("text/xml")
     def eml() {
         if (params.id) {
-            def pg = providerGroupService._get(params.id)
+            def pg = providerGroupService._getEager(params.id)
             if (pg) {
                 response.contentType = 'text/xml'
                 def xml = emlRenderService.emlForEntity(pg)
@@ -864,7 +912,7 @@ class DataController {
             // just do hubs to start
             int validCount = 0
             def invalid = []
-            //DataHub.list().each {
+
             Collection.list().each {
                 def xml = emlRenderService.emlForEntity(it)
                 def error = ''
@@ -1070,7 +1118,6 @@ class DataController {
         }
         def props = params.json
         props.userLastModified = session.username
-        //println "body = "  + props
         if (params.id) {
             // update
             def c = Contact.get(params.id)
@@ -1290,12 +1337,11 @@ class DataController {
         if (!ok) {
             return
         }
-        //println "notify"
+
         if (request.method != 'POST') {
             log.error("not allowed")
             notAllowed()
         } else {
-            //println params.json
             def payload = params.json
             def uid = payload.uid
             def event = payload.event
@@ -1305,7 +1351,6 @@ class DataController {
                 log.error("bad request")
                 badRequest 'must specify a uid, an event and an event id'
             } else {
-                //println "OK"
                 // register the event
                 activityLogService.log([user: 'notify-service', isAdmin: false, action: "${action}d ${id}", entityUid: uid])
                 success "notification accepted"
@@ -1406,7 +1451,6 @@ class DataController {
         }
         def props = params.json
         props.userLastModified = session.username
-        //println "body = "  + props
         def c = Contact.get(params.id)
         def cf = ContactFor.findByContactAndEntityUid(c, params.pg.uid)
         if (cf) {
diff --git a/grails-app/controllers/au/org/ala/collectory/DataLinkController.groovy b/grails-app/controllers/au/org/ala/collectory/DataLinkController.groovy
deleted file mode 100644
index 5b992731..00000000
--- a/grails-app/controllers/au/org/ala/collectory/DataLinkController.groovy
+++ /dev/null
@@ -1,108 +0,0 @@
-package au.org.ala.collectory
-
-class DataLinkController {
-
-    static allowedMethods = [save: "POST", update: "POST", delete: "POST"]
-
-    def index = {
-        redirect(action: "list", params: params)
-    }
-
-    def list = {
-        params.max = Math.min(params.max ? params.int('max') : 15, 1000)
-        if (params.consumer) {
-            def links = DataLink.findAllByConsumer(params.consumer)
-            [dataLinkInstanceList: links, dataLinkInstanceTotal: links.size(), consumer: params.consumer, returnTo: params.consumer]
-        } else if (params.provider) {
-            def links = DataLink.findAllByProvider(params.provider)
-            [dataLinkInstanceList: links, dataLinkInstanceTotal: links.size(), provider: params.provider, returnTo: params.provider]
-        } else {
-            [dataLinkInstanceList: DataLink.list(params), dataLinkInstanceTotal: DataLink.count(), returnTo: params.provider]
-        }
-    }
-
-    def create = {
-        def dataLinkInstance = new DataLink()
-        dataLinkInstance.properties = params
-        return [dataLinkInstance: dataLinkInstance, returnTo: params.returnTo]
-    }
-
-    def save = {
-        def dataLinkInstance = new DataLink(params)
-        if (dataLinkInstance.save(flush: true)) {
-            flash.message = "${message(code: 'default.created.message', args: [message(code: 'dataLink.label', default: 'DataLink'), dataLinkInstance.id])}"
-            redirect(action: "show", id: dataLinkInstance.id, params: [returnTo: params.returnTo])
-        }
-        else {
-            render(view: "create", model: [dataLinkInstance: dataLinkInstance, returnTo: params.returnTo])
-        }
-    }
-
-    def show = {
-        def dataLinkInstance = DataLink.get(params.id)
-        if (!dataLinkInstance) {
-            flash.message = "${message(code: 'default.not.found.message', args: [message(code: 'dataLink.label', default: 'DataLink'), params.id])}"
-            redirect(action: "list")
-        }
-        else {
-            [dataLinkInstance: dataLinkInstance, returnTo: params.returnTo]
-        }
-    }
-
-    def edit = {
-        def dataLinkInstance = DataLink.get(params.id)
-        if (!dataLinkInstance) {
-            flash.message = "${message(code: 'default.not.found.message', args: [message(code: 'dataLink.label', default: 'DataLink'), params.id])}"
-            redirect(action: "list")
-        }
-        else {
-            return [dataLinkInstance: dataLinkInstance, returnTo: params.returnTo]
-        }
-    }
-
-    def update = {
-        def dataLinkInstance = DataLink.get(params.id)
-        if (dataLinkInstance) {
-            if (params.version) {
-                def version = params.version.toLong()
-                if (dataLinkInstance.version > version) {
-
-                    dataLinkInstance.errors.rejectValue("version", "default.optimistic.locking.failure", [message(code: 'dataLink.label', default: 'DataLink')] as Object[], "Another user has updated this DataLink while you were editing")
-                    render(view: "edit", model: [dataLinkInstance: dataLinkInstance, returnTo: params.returnTo])
-                    return
-                }
-            }
-            dataLinkInstance.properties = params
-            if (!dataLinkInstance.hasErrors() && dataLinkInstance.save(flush: true)) {
-                flash.message = "${message(code: 'default.updated.message', args: [message(code: 'dataLink.label', default: 'DataLink'), dataLinkInstance.id])}"
-                redirect(action: "show", id: dataLinkInstance.id, params:[returnTo: params.returnTo])
-            }
-            else {
-                render(view: "edit", model: [dataLinkInstance: dataLinkInstance, returnTo: params.returnTo])
-            }
-        }
-        else {
-            flash.message = "${message(code: 'default.not.found.message', args: [message(code: 'dataLink.label', default: 'DataLink'), params.id])}"
-            redirect(action: "list", params:[returnTo: params.returnTo])
-        }
-    }
-
-    def delete = {
-        def dataLinkInstance = DataLink.get(params.id)
-        if (dataLinkInstance) {
-            try {
-                dataLinkInstance.delete(flush: true)
-                flash.message = "${message(code: 'default.deleted.message', args: [message(code: 'dataLink.label', default: 'DataLink'), params.id])}"
-                redirect(action: "list", params:[returnTo: params.returnTo])
-            }
-            catch (org.springframework.dao.DataIntegrityViolationException e) {
-                flash.message = "${message(code: 'default.not.deleted.message', args: [message(code: 'dataLink.label', default: 'DataLink'), params.id])}"
-                redirect(action: "show", id: params.id, params:[returnTo: params.returnTo])
-            }
-        }
-        else {
-            flash.message = "${message(code: 'default.not.found.message', args: [message(code: 'dataLink.label', default: 'DataLink'), params.id])}"
-            redirect(action: "list", params:[returnTo: params.returnTo])
-        }
-    }
-}
diff --git a/grails-app/controllers/au/org/ala/collectory/DataProviderController.groovy b/grails-app/controllers/au/org/ala/collectory/DataProviderController.groovy
index 480a78d5..b91c74a2 100644
--- a/grails-app/controllers/au/org/ala/collectory/DataProviderController.groovy
+++ b/grails-app/controllers/au/org/ala/collectory/DataProviderController.groovy
@@ -21,6 +21,7 @@ class DataProviderController extends ProviderGroupController {
         }
         params.max = Math.min(params.max ? params.int('max') : 10000, 10000)
         params.sort = params.sort ?: "name"
+        params.fetch = [resources: 'join']
         activityLogService.log username(), isAdmin(), Action.LIST
         [instanceList: DataProvider.list(params), entityType: 'DataProvider', instanceTotal: DataProvider.count()]
     }
@@ -172,25 +173,38 @@ class DataProviderController extends ProviderGroupController {
     def updateConsumers = {
         def pg = get(params.id)
         def newConsumers = params.consumers.tokenize(',')
-        def oldConsumers = pg.listConsumers()
+        def oldConsumers = (pg.consumerCollections + pg.consumerInstitutions).collect { it.uid }
         // create new links
-        newConsumers.each {
-            if (!(it in oldConsumers)) {
-                DataLink.withTransaction {
-                    def dl = new DataLink(consumer: it, provider: pg.uid).save()
-                    auditLog(pg, 'INSERT', 'consumer', '', it, dl)
-                    log.info "created link from ${pg.uid} to ${it}"
+        newConsumers.each { String nc ->
+            if (!(nc in oldConsumers)) {
+                DataProvider.withTransaction {
+                    if (nc[0..1] == 'co') {
+                        Collection c = Collection.findByUid(nc)
+                        pg.consumerCollections.add(c)
+                    } else {
+                        Institution i = Institution.findByUid(nc)
+                        pg.consumerInstitutions.add(i)
+                    }
+                    pg.save(flush: true)
+                    auditLog(pg, 'INSERT', 'consumer', '', nc, pg)
+                    log.info "created link from ${pg.uid} to ${nc}"
                 }
             }
         }
         // remove old links - NOTE only for the variety (collection or institution) that has been returned
-        oldConsumers.each {
-            if (!(it in newConsumers) && it[0..1] == params.source) {
-                DataLink.withTransaction {
-                    log.info "deleting link from ${pg.uid} to ${it}"
-                    def dl = DataLink.findByConsumerAndProvider(it, pg.uid)
-                    auditLog(pg, 'DELETE', 'consumer', it, '', dl)
-                    dl.delete()
+        oldConsumers.each { String oc ->
+            if (!(oc in newConsumers) && oc[0..1] == params.source) {
+                DataProvider.withTransaction {
+                    log.info "deleting link from ${pg.uid} to ${oc}"
+                    if (oc[0..1] == 'co') {
+                        Collection c = Collection.findByUid(oc)
+                        pg.consumerCollections.remove(c)
+                    } else {
+                        Institution i = Institution.findByUid(oc)
+                        pg.consumerInstitutions.remove(i)
+                    }
+                    pg.save(flush: true)
+                    auditLog(pg, 'DELETE', 'consumer', oc, '', pg)
                 }
             }
         }
diff --git a/grails-app/controllers/au/org/ala/collectory/DataResourceController.groovy b/grails-app/controllers/au/org/ala/collectory/DataResourceController.groovy
index 74a5ff8a..55ee9f0e 100644
--- a/grails-app/controllers/au/org/ala/collectory/DataResourceController.groovy
+++ b/grails-app/controllers/au/org/ala/collectory/DataResourceController.groovy
@@ -241,25 +241,41 @@ class DataResourceController extends ProviderGroupController {
     def updateConsumers = {
         def pg = get(params.id)
         def newConsumers = params.consumers.tokenize(',')
-        def oldConsumers = pg.listConsumers()
+        def oldConsumers = (pg.consumerCollections + pg.consumerInstitutions).collect { it.uid }
         // create new links
-        newConsumers.each {
-            if (!(it in oldConsumers)) {
-                DataLink.withTransaction { status ->
-                    def dl = new DataLink(consumer: it, provider: pg.uid).save()
-                    auditLog(pg, 'INSERT', 'consumer', '', it, dl)
-                    log.info "created link from ${pg.uid} to ${it}"
+        newConsumers.each { String nc ->
+            if (!(nc in oldConsumers)) {
+                DataResource.withTransaction {
+                    if (nc[0..1] == 'co') {
+                        Collection c = Collection.findByUid(nc)
+                        pg.consumerCollections.add(c)
+                        pg.save(flush: true)
+                    } else {
+                        Institution i = Institution.findByUid(nc)
+                        pg.consumerInstitutions.add(i)
+                        pg.save(flush: true)
+                    }
+                    auditLog(pg, 'INSERT', 'consumer', '', nc, pg)
+                    log.info "created link from ${pg.uid} to ${nc}"
                 }
             }
         }
         // remove old links - NOTE only for the variety (collection or institution) that has been returned
-        oldConsumers.each {
-            if (!(it in newConsumers) && it[0..1] == params.source) {
-                DataLink.withTransaction { status ->
-                    log.info "deleting link from ${pg.uid} to ${it}"
-                    def dl = DataLink.findByConsumerAndProvider(it, pg.uid)
-                    auditLog(pg, 'DELETE', 'consumer', it, '', dl)
-                    dl.delete()
+        oldConsumers.each { String oc ->
+            if (!(oc in newConsumers) && oc[0..1] == params.source) {
+                DataProvider.withTransaction {
+                    log.info "deleting link from ${pg.uid} to ${oc}"
+                    if (oc[0..1] == 'co') {
+                        Collection c = Collection.findByUid(oc)
+                        pg.consumerCollections.remove(c)
+                        pg.save(flush: true)
+                    } else {
+                        Institution i = Institution.findByUid(oc)
+                        pg.consumerInstitutions.remove(i)
+                        pg.save(flush: true)
+                    }
+                    auditLog(pg, 'DELETE', 'consumer', oc, '', pg)
+
                 }
             }
         }
diff --git a/grails-app/controllers/au/org/ala/collectory/GbifController.groovy b/grails-app/controllers/au/org/ala/collectory/GbifController.groovy
index 6d806d75..2d91f06d 100644
--- a/grails-app/controllers/au/org/ala/collectory/GbifController.groovy
+++ b/grails-app/controllers/au/org/ala/collectory/GbifController.groovy
@@ -31,6 +31,7 @@ class GbifController {
     def gbifService
     def authService
     def externalDataService
+    def dataLinkService
 
     def healthCheck() {
         gbifRegistryService.generateSyncBreakdown()
@@ -63,21 +64,15 @@ class GbifController {
                 dataResourcesWithData[dataResource] = result.count
 
                 //get the data provider if available...
-                def dataLinks = DataLink.findAllByProvider(uid)
-                def institutionDataLink
                 def linked = false
 
-                if(dataLinks){
-                    //do we have institution link ????
-                    institutionDataLink = dataLinks.find { it.consumer.startsWith("in")}
-                    if(institutionDataLink){
-                        //we have an institution
-                        linkedToInstitution[dataResource] = result.count
-                        linked = true
-                    }
+                if(dataResource.consumerInstitutions){
+                    //we have an institution
+                    linkedToInstitution[dataResource] = result.count
+                    linked = true
                 }
 
-                if(!institutionDataLink) {
+                if(!linked) {
                     def dataProvider = dataResource.getDataProvider()
                     if(!dataProvider){
                         isShareable = false //no institution and no data provider
diff --git a/grails-app/controllers/au/org/ala/collectory/LookupController.groovy b/grails-app/controllers/au/org/ala/collectory/LookupController.groovy
index 8cabae4e..4bf2566a 100644
--- a/grails-app/controllers/au/org/ala/collectory/LookupController.groovy
+++ b/grails-app/controllers/au/org/ala/collectory/LookupController.groovy
@@ -319,10 +319,10 @@ class LookupController {
     }
 
     def downloadLimits = {
-        def list = DataResource.list(sort: "uid").findAll({it.downloadLimit > 0}).collect {
-            [uid: it.uid,
-            name: it.name,
-            downloadLimit: it.downloadLimit]
+        def list = DataResource.executeQuery("select uid, name, downloadLimit from DataResource where downloadLimit > 0 order by uid").collect {
+            [uid: it[0],
+            name: it[1],
+            downloadLimit: it[2]]
         }
         render list as JSON
     }
diff --git a/grails-app/controllers/au/org/ala/collectory/ProviderGroupController.groovy b/grails-app/controllers/au/org/ala/collectory/ProviderGroupController.groovy
index c81bf69a..691bdf3f 100644
--- a/grails-app/controllers/au/org/ala/collectory/ProviderGroupController.groovy
+++ b/grails-app/controllers/au/org/ala/collectory/ProviderGroupController.groovy
@@ -45,7 +45,14 @@ abstract class ProviderGroupController {
      * List providers for institutions/collections
      */
     def showProviders = {
-        def provs = DataLink.findAllByConsumer(params.id).collect {it.provider}
+        def provs = []
+        if (params.id[0..1] == 'co') {
+            Collection c = Collection.findByUid(params.uid)
+            provs = c.providerDataProviders.collect { it.uid } + c.providerDataResources.collect { it.uid }
+        } else {
+            Institution c = Institution.findByUid(params.uid)
+            provs = c.providerDataProviders.collect { it.uid } + c.providerDataResources.collect { it.uid }
+        }
         render provs as JSON
     }
 
@@ -53,7 +60,14 @@ abstract class ProviderGroupController {
      * List consumers of data resources/providers
      */
     def showConsumers = {
-        def cons = DataLink.findAllByProvider(params.id).collect {it.consumer}
+        def cons = []
+        if (params.id[0..1] == 'dr') {
+            DataResource c = DataResource.findByUid(params.uid)
+            cons = c.consumerCollections.collect { it.uid } + c.consumerInstitutions.collect { it.uid }
+        } else {
+            DataProvider c = DataProvider.findByUid(params.uid)
+            cons = c.consumerCollections.collect { it.uid } + c.consumerInstitutions.collect { it.uid }
+        }
         render cons as JSON
     }
 
diff --git a/grails-app/controllers/au/org/ala/collectory/ReportsController.groovy b/grails-app/controllers/au/org/ala/collectory/ReportsController.groovy
index 44e00b76..dda7501f 100644
--- a/grails-app/controllers/au/org/ala/collectory/ReportsController.groovy
+++ b/grails-app/controllers/au/org/ala/collectory/ReportsController.groovy
@@ -1,6 +1,5 @@
 package au.org.ala.collectory
 
-import au.org.ala.collectory.AuditLogEvent
 import grails.converters.JSON
 
 class ReportsController {
@@ -307,7 +306,18 @@ class ReportsController {
     }
 
     def dataLinks = {
-        [links: DataLink.list([sort:'provider'])]
+        def links = (DataProvider.executeQuery("select dp.uid, c.uid from DataProvider dp inner join dp.consumerInstitutions c") +
+        DataProvider.executeQuery("select dp.uid, c.uid from DataProvider dp inner join dp.consumerCollections c")+
+        DataProvider.executeQuery("select dr.uid, c.uid from DataResource dr inner join dr.consumerInstitutions c")+
+        DataProvider.executeQuery("select dr.uid, c.uid from DataResource dr inner join dr.consumerCollections c")).collect {
+            [consumer: it[1], provider: it[0]]
+        }
+
+        def sorted = links.sort {a, b ->
+            a.provider.compareTo(b.provider)
+        }
+
+        [links: sorted]
     }
 
     class ReportCommand {
diff --git a/grails-app/domain/au/org/ala/collectory/Collection.groovy b/grails-app/domain/au/org/ala/collectory/Collection.groovy
index c4bef4c7..be36d6f5 100644
--- a/grails-app/domain/au/org/ala/collectory/Collection.groovy
+++ b/grails-app/domain/au/org/ala/collectory/Collection.groovy
@@ -142,10 +142,10 @@ class Collection implements ProviderGroup, Serializable {
         numRecordsDigitised()
         states(nullable:true)
         geographicDescription(nullable:true)
-        eastCoordinate(max:360.0, min:-360.0, scale:10)
-        westCoordinate(max:360.0, min:-360.0, scale:10)
-        northCoordinate(max:360.0, min:-360.0, scale:10)
-        southCoordinate(max:360.0, min:-360.0, scale:10)
+        eastCoordinate(max:360.0, min:-360.0, scale:10, precision: 13)
+        westCoordinate(max:360.0, min:-360.0, scale:10, precision: 13)
+        northCoordinate(max:360.0, min:-360.0, scale:10, precision: 13)
+        southCoordinate(max:360.0, min:-360.0, scale:10, precision: 13)
         startDate(nullable:true, maxSize:45)
         endDate(nullable:true, maxSize:45)
         kingdomCoverage(nullable:true, maxSize:1024,
@@ -383,4 +383,23 @@ class Collection implements ProviderGroup, Serializable {
         return ENTITY_TYPE
     }
 
+    def getProviderDataResources() {
+        def c = DataResource.createCriteria()
+        def result = c.list {
+            consumerCollections {
+                idEq(this.id)
+            }
+        }
+        return result
+    }
+
+    def getProviderDataProviders() {
+        def c = DataProvider.createCriteria()
+        def result = c.list {
+            consumerCollections {
+                idEq(this.id)
+            }
+        }
+        return result
+    }
 }
diff --git a/grails-app/domain/au/org/ala/collectory/DataLink.groovy b/grails-app/domain/au/org/ala/collectory/DataLink.groovy
deleted file mode 100644
index df67016a..00000000
--- a/grails-app/domain/au/org/ala/collectory/DataLink.groovy
+++ /dev/null
@@ -1,16 +0,0 @@
-package au.org.ala.collectory
-
-/**
- * Describes relationships between natural history institutions/collections and data providers/resources.
- */
-class DataLink {
-
-    String consumer
-    String provider
-
-    static constraints = {
-        consumer(blank:false)
-        provider(blank:false)
-    }
-
-}
diff --git a/grails-app/domain/au/org/ala/collectory/DataProvider.groovy b/grails-app/domain/au/org/ala/collectory/DataProvider.groovy
index 9dbc42ef..14772d72 100644
--- a/grails-app/domain/au/org/ala/collectory/DataProvider.groovy
+++ b/grails-app/domain/au/org/ala/collectory/DataProvider.groovy
@@ -7,7 +7,7 @@ class DataProvider implements ProviderGroup, Serializable {
 
     static auditable = [ignore: ['version','dateCreated','lastUpdated','userLastModified']]
 
-    static hasMany = [resources: DataResource, externalIdentifiers: ExternalIdentifier]
+    static hasMany = [resources: DataResource, externalIdentifiers: ExternalIdentifier, consumerInstitutions: Institution, consumerCollections: Collection]
 
     String hiddenJSON // web service only (non-UI) JSON; used by fieldcapture to store project data
 
@@ -31,6 +31,9 @@ class DataProvider implements ProviderGroup, Serializable {
         taxonomyHints type: "text"
         notes type: "text"
         networkMembership type: "text"
+
+        consumerInstitutions joinTable:[name:"data_provider_institution", key:'data_provider_id' ]
+        consumerCollections joinTable:[name:"data_provider_collection", key:'data_provider_id' ]
     }
 
     static constraints = {
@@ -116,14 +119,12 @@ class DataProvider implements ProviderGroup, Serializable {
             }
             dps.resources = list
         }
-        def consumers = listConsumers()
+        def consumers = consumerCollections + consumerInstitutions
         consumers.each {
-            if (it[0..1] == 'co') {
-                def pg = Collection.findByUid(it)
-                dps.relatedCollections << [uid: pg.uid, name: pg.name]
+            if (it.uid[0..1] == 'co') {
+                dps.relatedCollections << [uid: it.uid, name: it.name]
             } else {
-                def pg = Institution.findByUid(it)
-                dps.relatedInstitutions << [uid: pg.uid, name: pg.name]
+                dps.relatedInstitutions << [uid: it.uid, name: it.name]
             }
         }
         return dps
@@ -147,10 +148,10 @@ class DataProvider implements ProviderGroup, Serializable {
             return getPrimaryContact()
         }
         else {
-            for (con in listConsumers()) {
-                def related = Collection.findByUid(con) ?: Institution.findByUid(con)
-                if (related.inheritPrimaryContact()) {
-                    return related.inheritPrimaryContact()
+            for (con in consumerCollections + consumerInstitutions) {
+                def primary = con.inheritPrimaryContact()
+                if (primary) {
+                    return primary
                 }
             }
             return null
@@ -167,10 +168,10 @@ class DataProvider implements ProviderGroup, Serializable {
             return getPrimaryPublicContact()
         }
         else {
-            for (con in listConsumers()) {
-                def related = Collection.findByUid(con) ?: Institution.findByUid(con)
-                if (related.inheritPrimaryPublicContact()) {
-                    return related.inheritPrimaryPublicContact()
+            for (con in consumerCollections + consumerInstitutions) {
+                def publicContact = con.inheritPrimaryPublicContact()
+                if (publicContact) {
+                    return publicContact
                 }
             }
             return null
diff --git a/grails-app/domain/au/org/ala/collectory/DataResource.groovy b/grails-app/domain/au/org/ala/collectory/DataResource.groovy
index 6279ff0b..bd845833 100644
--- a/grails-app/domain/au/org/ala/collectory/DataResource.groovy
+++ b/grails-app/domain/au/org/ala/collectory/DataResource.groovy
@@ -49,9 +49,9 @@ class DataResource implements ProviderGroup, Serializable {
         dataCollectionProtocolDoc type: "text"
         suitableFor type: "text"
         suitableForOtherDetail type: "text"
-        dataProvider fetch: 'join'
-        institution fetch: 'join'
-        externalIdentifiers fetch: 'join'
+
+        consumerInstitutions joinTable:[name:"data_resource_institution", key:'data_resource_id' ]
+        consumerCollections joinTable:[name:"data_resource_collection", key:'data_resource_id' ]
     }
 
     String rights
@@ -104,7 +104,7 @@ class DataResource implements ProviderGroup, Serializable {
 
     String gbifDoi
 
-    static hasMany = [externalIdentifiers: ExternalIdentifier]
+    static hasMany = [externalIdentifiers: ExternalIdentifier, consumerInstitutions: Institution, consumerCollections: Collection]
 
     static constraints = {
         guid(nullable:true, maxSize:256)
@@ -245,7 +245,7 @@ class DataResource implements ProviderGroup, Serializable {
      * @return
      */
     boolean hasMappedCollections() {
-        return listConsumers().size() as boolean
+        return consumerInstitutions || consumerCollections
     }
 
     /**
diff --git a/grails-app/domain/au/org/ala/collectory/Institution.groovy b/grails-app/domain/au/org/ala/collectory/Institution.groovy
index 6073b449..859fdb44 100644
--- a/grails-app/domain/au/org/ala/collectory/Institution.groovy
+++ b/grails-app/domain/au/org/ala/collectory/Institution.groovy
@@ -106,11 +106,11 @@ class Institution implements ProviderGroup, Serializable {
     def listCollections() {
         List result = collections.collect {it}
         if (childInstitutions) {
-            childInstitutions.tokenize(' ').each {
-                def i = get(it)
-                if (i) {
-                    result.addAll i.listCollections()
-                }
+            def list = childInstitutions.tokenize(' ')
+            Collections.createCriteria().list (fetch: [collections: 'join']) {
+                in ('id', list)
+            }.each {
+                result.addAll it.listCollections()
             }
         }
         return result
@@ -186,7 +186,27 @@ class Institution implements ProviderGroup, Serializable {
 //    }
 
 
-    List<DataLink> getLinkedDataResources() {
-        return DataLink.findAllByConsumer(this.uid)
+    List<String> getLinkedDataResources() {
+        return providerDataProviders.collect { it.uid } + providerDataResources.collect { it.uid }
+    }
+
+    def getProviderDataResources() {
+        def c = DataResource.createCriteria()
+        def result = c.list {
+            consumerInstitutions {
+                idEq(this.id)
+            }
+        }
+        return result
+    }
+
+    def getProviderDataProviders() {
+        def c = DataProvider.createCriteria()
+        def result = c.list {
+            consumerInstitutions {
+                idEq(this.id)
+            }
+        }
+        return result
     }
 }
diff --git a/grails-app/migrations/5.1.0.1.sql b/grails-app/migrations/5.1.0.1.sql
new file mode 100644
index 00000000..bbda426e
--- /dev/null
+++ b/grails-app/migrations/5.1.0.1.sql
@@ -0,0 +1,44 @@
+CREATE TABLE `data_provider_collection` (
+    `data_provider_id` bigint NOT NULL,
+    `collection_id` bigint NOT NULL,
+    KEY `FKkfqinj4hhx1y9f4sug6krrkkv` (`data_provider_id`),
+    KEY `FKo69vyjeqitr73xst7cn8vkj32` (`collection_id`),
+    CONSTRAINT `FKkfqinj4hhx1y9f4sug6krrkkv` FOREIGN KEY (`data_provider_id`) REFERENCES `data_provider` (`id`),
+    CONSTRAINT `FKo69vyjeqitr73xst7cn8vkj32` FOREIGN KEY (`collection_id`) REFERENCES `collection` (`id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8;
+
+insert into data_provider_collection (collection_id, data_provider_id) select c.id, d.id from data_link dl inner join collection c on dl.consumer = c.uid inner join data_provider d on dl.provider = d.uid;
+
+CREATE TABLE `data_provider_institution` (
+     `data_provider_id` bigint NOT NULL,
+     `institution_id` bigint NOT NULL,
+     KEY `FKekweu6pqdjjg8lv8bm1mb9q8m` (`institution_id`),
+    KEY `FKtl4l7nyivmpsuhqks8c7vr05q` (`data_provider_id`),
+    CONSTRAINT `FKekweu6pqdjjg8lv8bm1mb9q8m` FOREIGN KEY (`institution_id`) REFERENCES `institution` (`id`),
+    CONSTRAINT `FKtl4l7nyivmpsuhqks8c7vr05q` FOREIGN KEY (`data_provider_id`) REFERENCES `data_provider` (`id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8;
+
+insert into data_provider_institution (institution_id, data_provider_id) select c.id, d.id from data_link dl inner join institution c on dl.consumer = c.uid inner join data_provider d on dl.provider = d.uid;
+
+
+CREATE TABLE `data_resource_institution` (
+     `data_resource_id` bigint NOT NULL,
+     `institution_id` bigint DEFAULT NULL,
+     KEY `FKs242vnare7999fp4glq0wvapu` (`institution_id`),
+    KEY `FKdvn1twqb3ngbkv75l8lo4exe3` (`data_resource_id`),
+    CONSTRAINT `FKdvn1twqb3ngbkv75l8lo4exe3` FOREIGN KEY (`data_resource_id`) REFERENCES `data_resource` (`id`),
+    CONSTRAINT `FKs242vnare7999fp4glq0wvapu` FOREIGN KEY (`institution_id`) REFERENCES `institution` (`id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8;
+
+insert into data_resource_institution (institution_id, data_resource_id) select c.id, d.id from data_link dl inner join institution c on dl.consumer = c.uid inner join data_resource d on dl.provider = d.uid;
+
+CREATE TABLE `data_resource_collection` (
+    `collection_id` bigint NOT NULL,
+    `data_resource_id` bigint NOT NULL,
+    KEY `FKdytubkqq8k13w53us6bap0ad8` (`data_resource_id`),
+    KEY `FKgmdx83aviwdxxd61wf4mustfl` (`collection_id`),
+    CONSTRAINT `FKdytubkqq8k13w53us6bap0ad8` FOREIGN KEY (`data_resource_id`) REFERENCES `data_resource` (`id`),
+    CONSTRAINT `FKgmdx83aviwdxxd61wf4mustfl` FOREIGN KEY (`collection_id`) REFERENCES `collection` (`id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8;
+
+insert into data_resource_collection (collection_id, data_resource_id) select c.id, d.id from data_link dl inner join collection c on dl.consumer = c.uid inner join data_resource d on dl.provider = d.uid;
diff --git a/grails-app/migrations/changelog.xml b/grails-app/migrations/changelog.xml
index f219dba3..5ac54ff6 100644
--- a/grails-app/migrations/changelog.xml
+++ b/grails-app/migrations/changelog.xml
@@ -435,5 +435,11 @@ liquibase.command.referencePassword: XXXXX
     <changeSet author="ALA Dev Team" id="20231122-00">
         <sqlFile path="5.1.0.sql" relativeToChangelogFile="true" encoding="ISO-8859-1" stripComments="1" />
     </changeSet>
+    <changeSet author="ALA Dev Team" id="20231123-00">
+        <sqlFile path="5.1.0.1.sql" relativeToChangelogFile="true" encoding="ISO-8859-1" stripComments="1" />
+    </changeSet>
+
+    <!-- Table DataLink is no longer required but keep in case of rollback. -->
+
 
 </databaseChangeLog>
diff --git a/grails-app/services/au/org/ala/collectory/CollectionService.groovy b/grails-app/services/au/org/ala/collectory/CollectionService.groovy
index dcacf22a..1c7afd0e 100644
--- a/grails-app/services/au/org/ala/collectory/CollectionService.groovy
+++ b/grails-app/services/au/org/ala/collectory/CollectionService.groovy
@@ -53,14 +53,11 @@ class CollectionService {
         cs.derivedInstCodes = collection.getListOfInstitutionCodesForLookup()
         cs.derivedCollCodes = collection.getListOfCollectionCodesForLookup()
         cs.hubMembership = dataHubService.listDataHubs().findAll {it.isCollectionMember(collection.uid)}.collect { [uid: it.uid, name: it.name] }
-        collection.listProviders().each {
-            def pg = Collection.findByUid(it)
-            if (pg) {
-                if (it[0..1] == 'dp') {
-                    cs.relatedDataProviders << [uid: pg.uid, name: pg.name]
-                } else {
-                    cs.relatedDataResources << [uid: pg.uid, name: pg.name]
-                }
+        (collection.providerDataResources + collection.providerDataProviders).each {
+            if (it.uid[0..1] == 'dp') {
+                cs.relatedDataProviders << [uid: it.uid, name: it.name]
+            } else {
+                cs.relatedDataResources << [uid: it.uid, name: it.name]
             }
         }
         return cs
diff --git a/grails-app/services/au/org/ala/collectory/CollectoryAuthService.groovy b/grails-app/services/au/org/ala/collectory/CollectoryAuthService.groovy
index 15972fa3..801609d1 100644
--- a/grails-app/services/au/org/ala/collectory/CollectoryAuthService.groovy
+++ b/grails-app/services/au/org/ala/collectory/CollectoryAuthService.groovy
@@ -131,11 +131,11 @@ class CollectoryAuthService{
     private static String getApiKey(params, HttpServletRequest request) {
         def apiKey = {
             // handle api keys if present in params
-            if (request.JSON && request.JSON.api_key) {
+            if (request.JSON && !(request.JSON instanceof List) && request.JSON.api_key) {
                 request.JSON.api_key
-            } else if (request.JSON && request.JSON.apiKey) {
+            } else if (request.JSON && !(request.JSON instanceof List) && request.JSON.apiKey) {
                 request.JSON.apiKey
-            } else if (request.JSON && request.JSON.Authorization) {
+            } else if (request.JSON && !(request.JSON instanceof List) && request.JSON.Authorization) {
                 request.JSON.Authorization
             } else if (params.api_key) {
                 params.api_key
diff --git a/grails-app/services/au/org/ala/collectory/CrudService.groovy b/grails-app/services/au/org/ala/collectory/CrudService.groovy
index 3cf0f90f..aae41b20 100644
--- a/grails-app/services/au/org/ala/collectory/CrudService.groovy
+++ b/grails-app/services/au/org/ala/collectory/CrudService.groovy
@@ -133,8 +133,8 @@ class CrudService {
 
                 // provider specific
                 dataResources = p.resources.briefEntity()
-                if (p.listConsumers()) {
-                    linkedRecordConsumers = formatEntitiesFromUids(p.listConsumers())
+                if (p.consumerCollections || p.consumerInstitutions) {
+                    linkedRecordConsumers = (p.consumerCollections + p.consumerInstitutions).collect { [name: it.name, uri: it.buildUri(), uid: it.uid] }
                 }
                 if (p.externalIdentifiers) {
                     externalIdentifiers = p.externalIdentifiers.formatExternalIdentifiers()
@@ -326,6 +326,7 @@ class CrudService {
                     uri = grailsApplication.config.grails.serverURL + "/data/dataResource/" + p.logoRef.file
                 }
             }
+            def consumers = p.consumerCollections + p.consumerInstitutions
             use (OutputFormat) {
                 networkMembership = p.networkMembership?.formatNetworkMembership()
                 hubMembership = dataHubService.listDataHubs()?.findAll {it.isDataResourceMember(p.uid)}?.formatHubMembership()
@@ -356,8 +357,8 @@ class CrudService {
                     riskAssessment = p.riskAssessment
                 }
                 contentTypes = p.contentTypes ? p.contentTypes.formatJSON() : []
-                if (p.listConsumers()) {
-                    linkedRecordConsumers = formatEntitiesFromUids(p.listConsumers())
+                if (consumers) {
+                    linkedRecordConsumers = consumers.collect { [name: it.name, uri: it.buildUri(), uid: it.uid] }
                 }
                 if (p.connectionParameters) {
                     def connParams =  p.connectionParameters.formatJSON()
@@ -570,6 +571,7 @@ class CrudService {
                     uri = grailsApplication.config.grails.serverURL + "/data/institution/" + p.logoRef.file
                 }
             }
+            def providers = p.providerDataResources + p.providerDataProviders
             use (OutputFormat) {
                 networkMembership = p.networkMembership?.formatNetworkMembership()
                 hubMembership = dataHubService.listDataHubs()?.findAll {it.isInstitutionMember(p.uid)}?.formatHubMembership()
@@ -583,8 +585,8 @@ class CrudService {
                 collections = p.collections.briefEntity()
                 parentInstitutions = p.listParents().briefEntity()
                 childInstitutions = p.listChildren().briefEntity()
-                if (p.listProviders()) {
-                    linkedRecordProviders = formatEntitiesFromUids(p.listProviders())
+                if (providers) {
+                    linkedRecordProviders = providers.collect { [name: it.name, uri: it.buildUri(), uid: it.uid] }
                 }
                 gbifRegistryKey = p.gbifRegistryKey
                 if (p.externalIdentifiers) {
@@ -661,6 +663,7 @@ class CrudService {
                     uri = grailsApplication.config.grails.serverURL + "/data/collection/" + p.imageRef.file
                 }
             }
+            def providers = p.providerDataResources + p.providerDataProviders
             use (OutputFormat) {
                 networkMembership = p.networkMembership?.formatNetworkMembership()
                 hubMembership = dataHubService.listDataHubs()?.findAll {it.isCollectionMember(p.uid)}?.formatHubMembership()
@@ -711,8 +714,8 @@ class CrudService {
                         lastUpdated = p.providerMap.lastUpdated
                     }
                 }
-                if (p.listProviders()) {
-                    linkedRecordProviders = formatEntitiesFromUids(p.listProviders())
+                if (providers) {
+                    linkedRecordProviders = providers.collect { [name: it.name, uri: it.buildUri(), uid: it.uid] }
                 }
                 gbifRegistryKey = p.gbifRegistryKey
                 if (p.externalIdentifiers) {
diff --git a/grails-app/services/au/org/ala/collectory/DataResourceService.groovy b/grails-app/services/au/org/ala/collectory/DataResourceService.groovy
index 04dcb2e0..c940913e 100644
--- a/grails-app/services/au/org/ala/collectory/DataResourceService.groovy
+++ b/grails-app/services/au/org/ala/collectory/DataResourceService.groovy
@@ -42,15 +42,12 @@ class DataResourceService {
         drs.downloadLimit = dataResource.downloadLimit
 
         drs.hubMembership = dataHubService.listDataHubs().findAll {it.isDataResourceMember(dataResource.uid)}.collect { [uid: it.uid, name: it.name] }
-        def consumers = dataResource.listConsumers()
+        def consumers = dataResource.consumerInstitutions + dataResource.consumerCollections
         consumers.each {
-            def pg = DataResource.findByUid(it)
-            if (pg) {
-                if (it[0..1] == 'co') {
-                    drs.relatedCollections << [uid: pg.uid, name: pg.name]
-                } else {
-                    drs.relatedInstitutions << [uid: pg.uid, name: pg.name]
-                }
+            if (it.uid[0..1] == 'co') {
+                drs.relatedCollections << [uid: it.uid, name: it.name]
+            } else {
+                drs.relatedInstitutions << [uid: it.uid, name: it.name]
             }
         }
         // for backward compatibility
diff --git a/grails-app/services/au/org/ala/collectory/EmlRenderService.groovy b/grails-app/services/au/org/ala/collectory/EmlRenderService.groovy
index c825dc7a..6c47731a 100644
--- a/grails-app/services/au/org/ala/collectory/EmlRenderService.groovy
+++ b/grails-app/services/au/org/ala/collectory/EmlRenderService.groovy
@@ -107,11 +107,15 @@ class EmlRenderService {
 
         /* associated parties */
         builder.associatedParty(ala(true))
-        pg.listConsumers().each { con ->
-            organisation(builder, 'associatedParty', providerGroupService._get(con), 'originator')
+        if (pg instanceof DataResource || pg instanceof DataProvider) {
+            (pg.consumerInstitutions + pg.consumerCollections).each { con ->
+                organisation(builder, 'associatedParty', con, 'originator')
+            }
         }
-        pg.listProviders().each { pro ->
-            organisation(builder, 'associatedParty', providerGroupService._get(pro), 'publisher')
+        if (pg instanceof Institution || pg instanceof Collection) {
+            (pg.providerDataResources + pg.providerDataProviders).each { pro ->
+                organisation(builder, 'associatedParty', pro, 'publisher')
+            }
         }
 
         /* pub date */
@@ -592,7 +596,7 @@ class EmlRenderService {
                         /* dateStamp, metadataLanguage, hierarchyLevel, resourceLogoUrl */
                         commonElements2 xml, pg
                         citation pg.citation?:''
-                        rights  pg.rights?:''                        
+                        rights  pg.rights?:''
                     }
                 }
             }
diff --git a/grails-app/services/au/org/ala/collectory/GbifRegistryService.groovy b/grails-app/services/au/org/ala/collectory/GbifRegistryService.groovy
index 394771b9..b9042dfe 100644
--- a/grails-app/services/au/org/ala/collectory/GbifRegistryService.groovy
+++ b/grails-app/services/au/org/ala/collectory/GbifRegistryService.groovy
@@ -222,15 +222,8 @@ class GbifRegistryService {
         if (!institution) {
 
             //get the data provider if available...
-            def dataLinks = DataLink.findAllByProvider(dataResource.uid)
-            def institutionDataLink
-
-            if (dataLinks) {
-                //do we have institution link ????
-                institutionDataLink = dataLinks.find { it.consumer.startsWith("in") }
-                if (institutionDataLink) {
-                    institution = Institution.findByUid(institutionDataLink.consumer)
-                }
+            if (dataResource.consumerInstitutions) {
+                institution = dataResource.consumerInstitutions.first()
             }
         }
 
@@ -354,12 +347,9 @@ class GbifRegistryService {
                         skipSync = true
                     }
 
-                    def dataLinks = DataLink.findAllByProvider(dp.uid)
-                    dataLinks.each { dataLink ->
-                        if (dataLink.consumer.startsWith("in")){
-                            skipSync = true
-                            log.warn("${resource.uid} is linked to an institution [${dataLink.consumer}]... not syncing  ")
-                        }
+                    if (dp.consumerInstitutions) {
+                        skipSync = true
+                        log.warn("${resource.uid} is linked to an institution [${dataLink.consumer}]... not syncing  ")
                     }
 
                     if (!skipSync) {
@@ -368,13 +358,9 @@ class GbifRegistryService {
                 }
             } else {
                 log.warn("Need to add syncing of resources for institution....via datalinks...")
-                def dataLinks = DataLink.findAllByConsumer(dp.uid)
-                if (dataLinks) {
-                    dataLinks.each { dataLink ->
-                        DataResource dr = DataResource.findByUid(dataLink.provider)
-                        if (dr) {
-                            syncDataResource(dr, dp.gbifRegistryKey)
-                        }
+                if (dp instanceof Collection || dp instanceof Institution) {
+                    dp.providerDataResource.each { DataResource dr ->
+                        syncDataResource(dr, dp.gbifRegistryKey)
                     }
                 }
             }
@@ -861,16 +847,11 @@ class GbifRegistryService {
                 def institution
 
                 //get the data provider if available...
-                def dataLinks = DataLink.findAllByProvider(uid)
-                def institutionDataLink
-
-                if (dataLinks){
-                    //do we have institution link ????
-                    institutionDataLink = dataLinks.find { it.consumer.startsWith("in")}
-                    if(institutionDataLink){
-                        //we have an institution
-                        institution = Institution.findByUid(institutionDataLink.consumer)
-                    }
+                def institutionDataLink = dataResource.consumerInstitutions
+
+                if(institutionDataLink){
+                    //we have an institution
+                    institution = institutionDataLink.first()
                 }
 
                 if(!institutionDataLink) {
@@ -1089,19 +1070,13 @@ class GbifRegistryService {
                     } else {
 
                         //get the data provider if available...
-                        def dataLinks = DataLink.findAllByProvider(uid)
-                        def institutionDataLink
+                        def institutionDataLink = dataResource.consumerInstitutions
 
-                        if (dataLinks) {
-                            //do we have institution link ????
-                            institutionDataLink = dataLinks.find { it.consumer.startsWith("in") }
-                            if (institutionDataLink) {
+                        if (institutionDataLink) {
+                            institution = institutionDataLink.first()
 
-                                institution = Institution.findByUid(institutionDataLink.consumer)
-
-                                //we have an institution
-                                linkedToInstitution[dataResource.uid] = institution
-                            }
+                            //we have an institution
+                            linkedToInstitution[dataResource.uid] = institution
                         }
                     }
 
diff --git a/grails-app/services/au/org/ala/collectory/InstitutionService.groovy b/grails-app/services/au/org/ala/collectory/InstitutionService.groovy
index 51aeaf41..40d2f5fd 100644
--- a/grails-app/services/au/org/ala/collectory/InstitutionService.groovy
+++ b/grails-app/services/au/org/ala/collectory/InstitutionService.groovy
@@ -40,15 +40,11 @@ class InstitutionService {
         is.institutionUid = institution.uid
         is.institutionName = institution.name
         is.collections = institution.collections.collect { [it.uid, it.name] }
-        institution.listProviders().each {
-            def pg = Institution.findByUid(it)
-            if (pg) {
-                if (it[0..1] == 'dp') {
-                    is.relatedDataProviders << [uid: pg.uid, name: pg.name]
-                } else {
-                    is.relatedDataResources << [uid: pg.uid, name: pg.name]
-                }
-            }
+        institution.providerDataProviders.each {
+            is.relatedDataProviders << [uid: it.uid, name: it.name]
+        }
+        institution.providerDataResources.each {
+            is.relatedDataResources << [uid: it.uid, name: it.name]
         }
         is.hubMembership = dataHubService.listDataHubs().findAll {it.isInstitutionMember(institution.uid)}.collect { [uid: it.uid, name: it.name] }
         return is
diff --git a/grails-app/services/au/org/ala/collectory/ProviderGroupService.groovy b/grails-app/services/au/org/ala/collectory/ProviderGroupService.groovy
index f5e1a878..8660d5d2 100644
--- a/grails-app/services/au/org/ala/collectory/ProviderGroupService.groovy
+++ b/grails-app/services/au/org/ala/collectory/ProviderGroupService.groovy
@@ -98,6 +98,30 @@ class ProviderGroupService {
         }
     }
 
+    ProviderGroup  _getEager(String uid) {
+        if (!uid || uid.size() < 3) {return null}
+        switch (uid[0..1]) {
+            case Institution.ENTITY_PREFIX:
+                return Institution.createCriteria().list([fetch: [collections: "join", providerMap: "join", externalIdentifiers: "join", attributions: "join"]]) {
+                    eq('uid', uid)
+                }[0]
+            case Collection.ENTITY_PREFIX:
+                return Collection.createCriteria().list([fetch: [institution: "join", providerMap: "join", externalIdentifiers: "join"]]) {
+                    eq('uid', uid)
+                }[0]
+            case DataProvider.ENTITY_PREFIX:
+                return DataProvider.createCriteria().list([fetch: [resources: "join", externalIdentifiers: "join", consumerInstitutions: "join", consumerCollections: "join"]]) {
+                    eq('uid', uid)
+                }[0]
+            case DataResource.ENTITY_PREFIX:
+                return DataResource.createCriteria().list([fetch: [dataProvider: "join", institution: "join", externalIdentifiers: "join", consumerInstitutions: "join", consumerCollections: "join"]]) {
+                    eq('uid', uid)
+                }[0]
+            case DataHub.ENTITY_PREFIX: return DataHub.findByUid(uid)
+            default: return null
+        }
+    }
+
     /**
      * Returns the instance identified by the uid.
      *
@@ -121,8 +145,8 @@ class ProviderGroupService {
 
     def resolveAddress(ProviderGroup pg){
         def addr = pg.resolveAddress()
-        if (!addr) {
-            pg.listConsumers().find {
+        if (!addr && (pg instanceof DataProvider || pg instanceof DataResource)) {
+            (pg.consumerInstitutions + pg.consumerCollections).find {
                 def related = _get(it)
                 return related && related.resolveAddress()
             }
diff --git a/grails-app/taglib/au/org/ala/collectory/CollectoryTagLib.groovy b/grails-app/taglib/au/org/ala/collectory/CollectoryTagLib.groovy
index 75536461..cdfb3ed5 100644
--- a/grails-app/taglib/au/org/ala/collectory/CollectoryTagLib.groovy
+++ b/grails-app/taglib/au/org/ala/collectory/CollectoryTagLib.groovy
@@ -1446,9 +1446,12 @@ class CollectoryTagLib {
         out << "<img class='editImg' style='margin-right:5px;vertical-align:middle' src='${resource(dir:'images/ala',file:'edit.png')}'/>\n"
         out << link(controller:attrs.instance.urlForm(), action:'show', id:attrs.instance.uid) {"Edit metadata"}
         out << " for this ${attrs.instance.textFormOfEntityType(attrs.instance.uid)}. You need<br/>appropriate authorisation to do this. You will<br/>be asked to log in if you are not already.</p>\n"
-        def providers = attrs.instance.listProviders()
-        if (attrs.instance instanceof Collection && attrs.instance.institution) {
-            providers += attrs.instance.institution?.listProviders()
+        def providers = []
+        if (attrs.instance instanceof Collection) {
+            providers = attrs.instance.providerDataProviders + attrs.instance.providerDataResources
+            if (attrs.instance.institution) {
+                providers += attrs.instance.institution.providerDataProviders + attrs.instance.institution.providerDataResources
+            }
         }
         if (providers) {
             boolean first = true
@@ -1458,18 +1461,18 @@ class CollectoryTagLib {
                     out << "<p class='viewList' style='margin-top:5px;'>View data sources<br/><ul>\n"
                     first = false
                 }
-                def provider = providerGroupService._get(it)
-                if (provider) {
-                    out << "<li>" +
-                            link(action:'show',id:provider.uid) {provider.name} +
-                            "</li>\n"
-                }
+                out << "<li>" +
+                        link(action:'show',id:it.uid) {it.name} +
+                        "</li>\n"
             }
             out << "</ul></p>\n"
         }
-        def consumers = attrs.instance.listConsumers()
-        if (attrs.instance instanceof DataResource && attrs.instance.dataProvider) {
-            consumers += attrs.instance.dataProvider?.listConsumers()
+        def consumers = []
+        if (attrs.instance instanceof DataProvider || attrs.instance instanceof DataResource) {
+            consumers += attrs.instance.consumerInstitutions + attrs.instance.consumerCollections
+            if (attrs.instance.dataProvider) {
+                consumers += attrs.instance.dataProvider.consumerInstitutions + attrs.instance.dataProvider.consumerCollections
+            }
         }
         if (consumers) {
             boolean first = true
@@ -1479,12 +1482,9 @@ class CollectoryTagLib {
                     out << "<p class='viewList' style='margin-top:5px;'>View data consumers<br/><ul>\n"
                     first = false
                 }
-                def consumer = providerGroupService._get(it)
-                if (consumer) {
-                    out << "<li>" +
-                            link(action:'show',id:consumer.uid) {consumer.name} +
-                            "</li>\n"
-                }
+                out << "<li>" +
+                        link(action:'show',id:it.uid) {it.name} +
+                        "</li>\n"
             }
             out << "</ul></p>\n"
         }
diff --git a/grails-app/views/dataLink/create.gsp b/grails-app/views/dataLink/create.gsp
deleted file mode 100644
index ac47af57..00000000
--- a/grails-app/views/dataLink/create.gsp
+++ /dev/null
@@ -1,44 +0,0 @@
-<%@ page import="au.org.ala.collectory.Collection; au.org.ala.collectory.Institution; au.org.ala.collectory.DataProvider; au.org.ala.collectory.DataResource; au.org.ala.collectory.DataLink" %>
-<html>
-    <head>
-        <meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
-        <meta name="layout" content="${grailsApplication.config.skin.layout}" />
-        <g:set var="entityName" value="${message(code: 'dataLink.label', default: 'DataLink')}" />
-        <title><g:message code="default.create.label" args="[entityName]" /></title>
-    </head>
-    <body>
-        <div class="btn-toolbar">
-            <ul class="btn-group">
-                <li class="btn btn-default"><cl:homeLink/></li>
-                <li class="btn btn-default"><span class="glyphicon glyphicon-list"></span><g:link class="list" action="list"> <g:message code="default.list.label" args="[entityName]"/></g:link></li>
-                <g:if test="${returnTo}"><li><cl:returnLink uid="${returnTo}"/></li></g:if>
-            </ul>
-        </div>
-        <div class="body">
-            <h1><g:message code="default.create.label" args="[entityName]" /></h1>
-            <g:if test="${flash.message}">
-            <div class="message">${flash.message}</div>
-            </g:if>
-            <g:hasErrors bean="${dataLinkInstance}">
-            <div class="errors">
-                <g:renderErrors bean="${dataLinkInstance}" as="list" />
-            </div>
-            </g:hasErrors>
-            <g:form action="save" >
-                <g:hiddenField name="returnTo" value="${returnTo}"/>
-                <div class="form-group ${hasErrors(bean: dataLinkInstance, field: 'consumer', 'errors')}">
-                    <label for="consumer"><g:message code="dataLink.consumer.label" default="Consumer" /></label>
-                    <g:select class="form-control" from="${Collection.list([sort:'name']) + Institution.list([sort:'name'])}" optionKey="uid" name="consumer" value="${dataLinkInstance?.consumer}" noSelection="['':'Choose a collection or Institution']"/>
-
-                </div>
-                <div class="form-group ${hasErrors(bean: dataLinkInstance, field: 'provider', 'errors')}">
-                    <label for="provider"><g:message code="dataLink.provider.label" default="Provider" /></label>
-                    <g:select class="form-control" from="${DataResource.list([sort:'name']) + DataProvider.list([sort:'name'])}" optionKey="uid" name="provider" value="${dataLinkInstance?.provider}" noSelection="['':'Choose a data provider or resource']"/>
-                </div>
-                <div class="buttons">
-                    <span class="button"><g:submitButton name="create" class="save btn btn-success" value="${message(code: 'default.button.create.label', default: 'Create')}" /></span>
-                </div>
-            </g:form>
-        </div>
-    </body>
-</html>
diff --git a/grails-app/views/dataLink/edit.gsp b/grails-app/views/dataLink/edit.gsp
deleted file mode 100644
index 0f86feb0..00000000
--- a/grails-app/views/dataLink/edit.gsp
+++ /dev/null
@@ -1,52 +0,0 @@
-<%@ page import="au.org.ala.collectory.DataResource; au.org.ala.collectory.DataProvider; au.org.ala.collectory.Collection; au.org.ala.collectory.Institution; au.org.ala.collectory.ProviderGroupService; au.org.ala.collectory.DataLink" %>
-<%
-    def providerGroupService = grailsApplication.classLoader.loadClass('au.org.ala.collectory.ProviderGroupService').newInstance()
-%>
-<html>
-    <head>
-        <meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
-        <meta name="layout" content="${grailsApplication.config.skin.layout}" />
-        <g:set var="entityName" value="${message(code: 'dataLink.label', default: 'DataLink')}" />
-        <title><g:message code="default.edit.label" args="[entityName]" /></title>
-    </head>
-    <body>
-        <div class="btn-toolbar">
-            <ul class="btn-group">
-                <li class="btn btn-default"><cl:homeLink/></li>
-                <li class="btn btn-default"><span class="glyphicon glyphicon-list"></span><g:link class="list" action="list"> <g:message code="default.list.label" args="[entityName]"/></g:link></li>
-                <li class="btn btn-default"><span class="glyphicon glyphicon-plus"></span><g:link class="create" action="create"> <g:message code="default.new.label" args="[entityName]"/></g:link></li>
-                <g:if test="${returnTo}"><li><cl:returnLink uid="${returnTo}"/></li></g:if>
-            </ul>
-        </div>
-        <div class="body">
-            <h1><g:message code="default.edit.label" args="[entityName]" /></h1>
-            <g:if test="${flash.message}">
-            <div class="message">${flash.message}</div>
-            </g:if>
-            <g:hasErrors bean="${dataLinkInstance}">
-            <div class="errors">
-                <g:renderErrors bean="${dataLinkInstance}" as="list" />
-            </div>
-            </g:hasErrors>
-            <g:form method="post" >
-                <g:hiddenField name="id" value="${dataLinkInstance?.id}" />
-                <g:hiddenField name="version" value="${dataLinkInstance?.version}" />
-                <g:hiddenField name="returnTo" value="${returnTo}"/>
-                <g:set var="provider" value="${providerGroupService._get(dataLinkInstance.provider)}"/>
-                <g:set var="consumer" value="${providerGroupService._get(dataLinkInstance.consumer)}"/>
-                <div class="form-group ${hasErrors(bean: dataLinkInstance, field: 'consumer', 'errors')}">
-                    <label for="consumer"><g:message code="dataLink.consumer.label" default="Consumer" /></label>
-                    <g:select class="form-control" from="${Collection.list([sort:'name']) + Institution.list([sort:'name'])}" optionKey="uid" name="consumer" value="${consumer.uid}"/>
-                </div>
-                <div class="form-group ${hasErrors(bean: dataLinkInstance, field: 'provider', 'errors')}">
-                    <label for="provider"><g:message code="dataLink.provider.label" default="Provider" /></label>
-                    <g:select class="form-control" from="${DataResource.list([sort:'name']) + DataProvider.list([sort:'name'])}" optionKey="uid" name="provider" value="${provider.uid}"/>
-                </div>
-                <div class="buttons">
-                    <span class="button"><g:actionSubmit class="save btn btn-success" action="update" value="${message(code: 'default.button.update.label', default: 'Update')}" /></span>
-                    <span class="button"><g:actionSubmit class="delete brn btn-danger" action="delete" value="${message(code: 'default.button.delete.label', default: 'Delete')}" onclick="return confirm('${message(code: 'default.button.delete.confirm.message', default: 'Are you sure?')}');" /></span>
-                </div>
-            </g:form>
-        </div>
-    </body>
-</html>
diff --git a/grails-app/views/dataLink/list.gsp b/grails-app/views/dataLink/list.gsp
deleted file mode 100644
index 8028e12d..00000000
--- a/grails-app/views/dataLink/list.gsp
+++ /dev/null
@@ -1,65 +0,0 @@
-
-<%@ page import="au.org.ala.collectory.ProviderGroup; au.org.ala.collectory.DataLink" %>
-<html>
-    <head>
-
-        <meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
-        <meta name="layout" content="${grailsApplication.config.skin.layout}" />
-        <g:set var="entityName" value="${message(code: 'dataLink.label', default: 'DataLink')}" />
-        <title><g:message code="default.list.label" args="[entityName]" /></title>
-    </head>
-    <body>
-        <div class="btn-toolbar">
-            <ul class="btn-group">
-                <li class="btn btn-default"><cl:homeLink/></li>
-                <li class="btn btn-default"><span class="glyphicon glyphicon-plus"></span><g:link class="create" action="create"> <g:message code="default.new.label" args="[entityName]"/></g:link></li>
-                <g:if test="${returnTo}"><li><cl:returnLink uid="${returnTo}"/></li></g:if>
-            </ul>
-        </div>
-        <div class="body">
-          <g:set var="filter" value="${consumer ? consumer : provider}"/>
-          <g:if test="${filter}">
-            <h1>DataLinks for ${providerGroupService._get(filter).name}</h1>
-          </g:if>
-          <g:else>
-            <h1><g:message code="default.list.label" args="[entityName]" /></h1>
-          </g:else>
-            <g:if test="${flash.message}">
-            <div class="message">${flash.message}</div>
-            </g:if>
-            <div class="list">
-                <table>
-                    <thead>
-                        <tr>
-                        
-                            <g:sortableColumn property="id" title="${message(code: 'dataLink.id.label', default: 'Id')}" />
-                        
-                            <g:sortableColumn property="consumer" title="${message(code: 'dataLink.consumer.label', default: 'Consumer')}" />
-                        
-                            <g:sortableColumn property="provider" title="${message(code: 'dataLink.provider.label', default: 'Provider')}" />
-                        
-                        </tr>
-                    </thead>
-                    <tbody>
-                    <g:each in="${dataLinkInstanceList}" status="i" var="dataLinkInstance">
-                        <g:set var="provider" value="${providerGroupService._get(dataLinkInstance.provider)}"/>
-                        <g:set var="consumer" value="${providerGroupService._get(dataLinkInstance.consumer)}"/>
-                        <tr class="${(i % 2) == 0 ? 'odd' : 'even'}">
-                        
-                            <td><g:link action="show" id="${dataLinkInstance.id}" params="[returnTo: returnTo]">${fieldValue(bean: dataLinkInstance, field: "id")}</g:link></td>
-                        
-                            <td>${fieldValue(bean: consumer, field: "name")}</td>
-                        
-                            <td>${fieldValue(bean: provider, field: "name")}</td>
-                        
-                        </tr>
-                    </g:each>
-                    </tbody>
-                </table>
-            </div>
-            <div class="nav">
-                <tb:paginate controller="dataLink" action="list" total="${dataLinkInstanceTotal}" />
-            </div>
-        </div>
-    </body>
-</html>
diff --git a/grails-app/views/dataLink/show.gsp b/grails-app/views/dataLink/show.gsp
deleted file mode 100644
index 029481fe..00000000
--- a/grails-app/views/dataLink/show.gsp
+++ /dev/null
@@ -1,67 +0,0 @@
-<%@ page import="au.org.ala.collectory.ProviderGroupService; au.org.ala.collectory.DataLink" %>
-<%
-    def providerGroupService = grailsApplication.classLoader.loadClass('au.org.ala.collectory.ProviderGroupService').newInstance()
-%>
-<html>
-    <head>
-        <meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
-        <meta name="layout" content="${grailsApplication.config.skin.layout}" />
-        <g:set var="entityName" value="${message(code: 'dataLink.label', default: 'DataLink')}" />
-        <title><g:message code="default.show.label" args="[entityName]" /></title>
-    </head>
-    <body>
-        <div class="btn-toolbar">
-            <ul class="btn-group">
-                <li class="btn btn-default"><cl:homeLink/></li>
-                <li class="btn btn-default"><span class="glyphicon glyphicon-list"></span><g:link class="list" action="list"> <g:message code="default.list.label" args="[entityName]"/></g:link></li>
-                <li class="btn btn-default"><span class="glyphicon glyphicon-plus"></span><g:link class="create" action="create"> <g:message code="default.new.label" args="[entityName]"/></g:link></li>
-                <g:if test="${returnTo}"><li><cl:returnLink uid="${returnTo}"/></li></g:if>
-            </ul>
-        </div>
-        <div class="body">
-            <h1><g:message code="default.show.label" args="[entityName]" /></h1>
-            <g:if test="${flash.message}">
-            <div class="message">${flash.message}</div>
-            </g:if>
-            <g:set var="provider" value="${providerGroupService._get(dataLinkInstance.provider)}"/>
-            <g:set var="consumer" value="${providerGroupService._get(dataLinkInstance.consumer)}"/>
-            <div class="dialog">
-                <table>
-                    <colgroup><col width="15%"><col width="60%"><col width="25%"></colgroup>
-                    <tbody>
-                    
-                        <tr class="prop">
-                            <td valign="top" class="name"><g:message code="dataLink.id.label" default="Id" /></td>
-                            
-                            <td valign="top" class="value">${fieldValue(bean: dataLinkInstance, field: "id")}</td>
-
-                        </tr>
-                    
-                        <tr class="prop">
-                            <td valign="top" class="name"><g:message code="dataLink.consumer.label" default="Consumer" /></td>
-                            
-                            <td valign="top" class="value">${fieldValue(bean: consumer, field: "name")}
-                            <g:link controller="${cl.controllerFromUid(uid:consumer.uid)}" action="show" id="${consumer.uid}">(${consumer.uid})</g:link></td>
-                        </tr>
-                    
-                        <tr class="prop">
-                            <td valign="top" class="name"><g:message code="dataLink.provider.label" default="Provider" /></td>
-                            
-                            <td valign="top" class="value">${fieldValue(bean: provider, field: "name")}
-                            <g:link controller="${cl.controllerFromUid(uid:provider.uid)}" action="show" id="${provider.uid}">(${provider.uid})</g:link></td>
-                        </tr>
-                    
-                    </tbody>
-                </table>
-            </div>
-            <div class="buttons">
-                <g:form>
-                    <g:hiddenField name="id" value="${dataLinkInstance?.id}" />
-                    <g:hiddenField name="returnTo" value="${returnTo}"/>
-                    <span class="button"><g:actionSubmit class="edit btn btn-default" action="edit" value="${message(code: 'default.button.edit.label', default: 'Edit')}" /></span>
-                    <span class="button"><g:actionSubmit class="delete btn btn-danger" action="delete" value="${message(code: 'default.button.delete.label', default: 'Delete')}" onclick="return confirm('${message(code: 'default.button.delete.confirm.message', default: 'Are you sure?')}');" /></span>
-                </g:form>
-            </div>
-        </div>
-    </body>
-</html>
diff --git a/grails-app/views/shared/_consumers.gsp b/grails-app/views/shared/_consumers.gsp
index 53e4ea7e..77bb7c3e 100644
--- a/grails-app/views/shared/_consumers.gsp
+++ b/grails-app/views/shared/_consumers.gsp
@@ -9,13 +9,11 @@
     <g:message code="shared.consumers.des02" />.
   </p>
   <ul class="fancy">
-    <g:each in="${instance.listConsumers()}" var="con">
-      <g:set var="pg" value="${providerGroupService._get(con)}"/>
-      <g:if test="${pg}">
-        <li><g:link controller="${cl.controllerFromUid(uid:con)}" action="show" id="${con}">${pg.name}</g:link> (${con[0..1] == 'in' ? 'institution' : 'collection'})</li>
-      </g:if>
-      <g:else><li><g:message code="shared.consumers.des03" />!</li></g:else>
-    </g:each>
+    <g:if test="${instance instanceof au.org.ala.collectory.DataProvider || instance instanceof au.org.ala.collectory.DataResource}">
+      <g:each in="${instance.consumerInstitutions + instance.consumerCollections}" var="pg">
+        <li><g:link controller="${cl.controllerFromUid(uid:pg.uid)}" action="show" id="${pg.uid}">${pg.name}</g:link> (${pg.uid[0..1] == 'in' ? 'institution' : 'collection'})</li>
+      </g:each>
+    </g:if>
   </ul>
   <div style="clear:both;"></div>
   <div>
diff --git a/grails-app/views/shared/_providers.gsp b/grails-app/views/shared/_providers.gsp
index 7d8e9914..5479b701 100644
--- a/grails-app/views/shared/_providers.gsp
+++ b/grails-app/views/shared/_providers.gsp
@@ -6,11 +6,10 @@
   <h2><g:message code="shared.providers.title01" /></h2>
   <p><g:message code="shared.providers.des01" args="[providerGroupService.textFormOfEntityType(instance.uid)]" />.</p>
   <ul class="fancy">
-    <g:each in="${instance.listProviders()}" var="prov">
-      <g:set var="pg" value="${providerGroupService._get(prov)}"/>
-      <g:if test="${pg}">
-        <g:set var="isProvider" value="${prov[0..1] == 'dp'}"/>
-        <li><g:link controller="${cl.controllerFromUid(uid:prov)}" action="show" id="${prov}">${pg.name}</g:link> (${isProvider ? 'provider' : 'resource'})</li>
+    <g:if test="${instance instanceof au.org.ala.collectory.Collection || instance instanceof au.org.ala.collectory.Institution}">
+      <g:each in="${instance.providerDataProviders + instance.providerDataResources}" var="pg">
+        <g:set var="isProvider" value="${pg.uid[0..1] == 'dp'}"/>
+        <li><g:link controller="${cl.controllerFromUid(uid:pg.uid)}" action="show" id="${pg.uid}">${pg.name}</g:link> (${isProvider ? 'provider' : 'resource'})</li>
         <g:if test="${isProvider}">
           <!-- list resources -->
           <ul class='resources'>
@@ -19,17 +18,13 @@
             </g:each>
           </ul>
         </g:if>
-      </g:if>
-      <g:else><li><g:message code="shared.providers.li01" />!</li></g:else>
-    </g:each>
+        <g:else><li><g:message code="shared.providers.li01" />!</li></g:else>
+      </g:each>
+    </g:if>
     <!-- for collections try their institution -->
     <g:if test="${instance instanceof au.org.ala.collectory.Collection && instance.institution}">
-      <g:each in="${instance.institution.listProviders()}" var="prov">
-        <g:set var="pg" value="${providerGroupService._get(prov)}"/>
-        <g:if test="${pg}">
-          <li><g:link controller="${cl.controllerFromUid(uid:prov)}" action="show" id="${prov}">${pg.name}</g:link> (${isProvider ? 'provider' : 'resource'}) - (via the institution)</li>
-        </g:if>
-        <g:else><li><g:message code="shared.providers.li01" />!</li></g:else>
+      <g:each in="${instance.institution.providerDataProviders + instance.institution.providerDataResources}" var="pg">
+        <li><g:link controller="${cl.controllerFromUid(uid:pg.uid)}" action="show" id="${pg.uid}">${pg.name}</g:link> (${isProvider ? 'provider' : 'resource'}) - (via the institution)</li>
       </g:each>
     </g:if>
   </ul>
diff --git a/src/main/groovy/au/org/ala/collectory/ProviderGroup.groovy b/src/main/groovy/au/org/ala/collectory/ProviderGroup.groovy
index 05c7c475..d8e62a1c 100644
--- a/src/main/groovy/au/org/ala/collectory/ProviderGroup.groovy
+++ b/src/main/groovy/au/org/ala/collectory/ProviderGroup.groovy
@@ -396,14 +396,14 @@ trait ProviderGroup implements Serializable {
      * @return list of Attribution
      */
     List<Attribution> getAttributionList() {
-        def uids = attributions?:''.tokenize(' ')
+        def uids = (attributions?:'').tokenize(' ')
         List<Attribution> list = []
-        uids.each {
-            def att = Attribution.findByUid(it as String)
-            if (att) {
-                list << att
+        if (uids) {
+            list = Attribution.createCriteria().list {
+                in('uid', uids)
             }
         }
+
         return list
     }
 
@@ -423,7 +423,7 @@ trait ProviderGroup implements Serializable {
     }
 
     void removeAttribution(String attributionUid) {
-        def uids = attributions.tokenize(' ')
+        def uids = (attributions?:'').tokenize(' ')
         uids.remove attributionUid
         attributions = uids.join(' ')
     }
@@ -450,7 +450,11 @@ trait ProviderGroup implements Serializable {
      * @return
      */
     List<String> listProviders() {
-        DataLink.findAllByConsumer(this.uid).collect {it.provider}
+        if (this instanceof Collection || this instanceof Institution) {
+            (this.consumerInstitutions + this.consumerCollections).collect { it.uid }
+        } else {
+            []
+        }
     }
 
     /**
@@ -458,7 +462,11 @@ trait ProviderGroup implements Serializable {
      * @return
      */
     def List<String> listConsumers() {
-        DataLink.findAllByProvider(this.uid).collect {it.consumer}
+        if (this instanceof DataProvider || this instanceof DataResource) {
+            (this.consumerInstitutions + this.consumerCollections).collect { it.uid }
+        } else {
+            []
+        }
     }
 
     /**

From 8319bf669a7b2346fb562e1d52671a65494c020c Mon Sep 17 00:00:00 2001
From: Adam Collins <adam.collins@csiro.au>
Date: Wed, 29 Nov 2023 13:55:34 +1000
Subject: [PATCH 06/13] openapi

---
 .../org/ala/collectory/DataController.groovy  | 48 +++++++++++++++++++
 1 file changed, 48 insertions(+)

diff --git a/grails-app/controllers/au/org/ala/collectory/DataController.groovy b/grails-app/controllers/au/org/ala/collectory/DataController.groovy
index 425e1a57..2ac0ec5d 100644
--- a/grails-app/controllers/au/org/ala/collectory/DataController.groovy
+++ b/grails-app/controllers/au/org/ala/collectory/DataController.groovy
@@ -493,6 +493,54 @@ class DataController {
         }
     }
 
+    @Operation(
+            method = "POST",
+            tags = "collection, institution, dataProvider, dataResource, tempDataResource, dataHub",
+            operationId = "getEntities",
+            summary = "Get an entity for a list of entity uids",
+            description = "Get detailed information for a list of entities",
+            parameters = [
+                    @Parameter(
+                            name = "entity",
+                            in = PATH,
+                            description = "entity type; collection, institution, dataProvider, dataResource, tempDataResource, dataHub",
+                            schema = @Schema(implementation = String),
+                            example = "collection",
+                            required = true
+                    ),
+                    @Parameter(
+                            name = "apikey",
+                            in = HEADER,
+                            description = "authorisation for dataResource connection details",
+                            schema = @Schema(implementation = String),
+                            required = false
+                    )
+            ],
+            requestBody = @RequestBody(
+                    description = "List of uids as JSON list",
+                    content = [
+                        @Content(mediaType = "application/json")
+                    ]
+            ),
+            responses = [
+                    @ApiResponse(
+                            description = "Entity Info",
+                            responseCode = "200",
+                            content = [
+                                    @Content(
+                                            mediaType = "application/json",
+                                            array = @ArraySchema(schema = @Schema(oneOf = [Collection, Institution, DataProvider, DataResource, TempDataResource, DataHub]))
+                                    )
+                            ],
+                            headers = [
+                                    @Header(name = 'Access-Control-Allow-Headers', description = "CORS header", schema = @Schema(type = "String")),
+                                    @Header(name = 'Access-Control-Allow-Methods', description = "CORS header", schema = @Schema(type = "String")),
+                                    @Header(name = 'Access-Control-Allow-Origin', description = "CORS header", schema = @Schema(type = "String"))
+                            ]
+                    )
+            ],
+            security = []
+    )
     @Path("/ws/{entity}")
     @Produces("application/json")
     def getEntities() {

From 584c4e6211fe92918927d0b0e6b400723b189fb4 Mon Sep 17 00:00:00 2001
From: Adam Collins <adam.collins@csiro.au>
Date: Fri, 1 Dec 2023 09:10:11 +1000
Subject: [PATCH 07/13] fix buildSummary usage

---
 .../collectory/CollectionController.groovy    |  6 ++---
 .../ala/collectory/LookupController.groovy    | 22 +++++++++----------
 .../ala/collectory/ReportsController.groovy   |  8 +++----
 3 files changed, 18 insertions(+), 18 deletions(-)

diff --git a/grails-app/controllers/au/org/ala/collectory/CollectionController.groovy b/grails-app/controllers/au/org/ala/collectory/CollectionController.groovy
index 8c1fdd24..ffccbbc0 100644
--- a/grails-app/controllers/au/org/ala/collectory/CollectionController.groovy
+++ b/grails-app/controllers/au/org/ala/collectory/CollectionController.groovy
@@ -115,7 +115,7 @@ class CollectionController extends ProviderGroupController {
     def summary = {
         Collection collectionInstance = findCollection(params.id)
         if (collectionInstance) {
-            render collectionService.buildSummary(collectionInstance.uid) as JSON
+            render collectionService.buildSummary(collectionInstance) as JSON
         } else {
             log.error "Unable to find collection for id = ${params.id}"
             def error = ["error":"unable to find collection for id = " + params.id]
@@ -144,10 +144,10 @@ class CollectionController extends ProviderGroupController {
         ProviderGroup pg = ProviderMap.findMatch(inst, coll)
         if (pg) {
             if (pg[0..1] == 'co') {
-                render collectionService.buildSummary(pg.uid) as JSON
+                render collectionService.buildSummary(pg) as JSON
             } else {
                 // institution
-                render institutionService.buildSummary(pg.uid) as JSON
+                render institutionService.buildSummary(pg) as JSON
             }
         } else {
             def error = ["error":"unable to find collection with inst code = ${inst} and coll code = ${coll}"]
diff --git a/grails-app/controllers/au/org/ala/collectory/LookupController.groovy b/grails-app/controllers/au/org/ala/collectory/LookupController.groovy
index 4bf2566a..bce08e5e 100644
--- a/grails-app/controllers/au/org/ala/collectory/LookupController.groovy
+++ b/grails-app/controllers/au/org/ala/collectory/LookupController.groovy
@@ -101,10 +101,10 @@ class LookupController {
             ProviderGroup pg = ProviderMap.findMatch(inst, coll)
             if (pg) {
                 if (pg.uid[0..1] == 'co') {
-                    render collectionService.buildSummary(pg.uid) as JSON
+                    render collectionService.buildSummary(pg) as JSON
                 } else {
                     // institution
-                    render institutionService.buildSummary(pg.uid) as JSON
+                    render institutionService.buildSummary(pg) as JSON
                 }
             } else {
                 def error = ["error":"unable to find collection with inst code = ${inst} and coll code = ${coll}"]
@@ -145,11 +145,11 @@ class LookupController {
             pg = providerGroupService._get(uid)
             if (pg) {
                 if (pg.uid[0..1] == 'co') {
-                    render collectionService.buildSummary(pg.uid) as JSON
+                    render collectionService.buildSummary(pg) as JSON
                 } else if (pg[0..1] == 'in') {
-                    render institutionService.buildSummary(pg.uid) as JSON
+                    render institutionService.buildSummary(pg) as JSON
                 } else if (pg[0..1] == 'dr') {
-                    render dataResourceService.buildSummary(pg.uid) as JSON
+                    render dataResourceService.buildSummary(pg) as JSON
                 } else {
                     render pg.buildSummary() as JSON
                 }
@@ -163,11 +163,11 @@ class LookupController {
             def list = []
             domain.list().each {
                 if (it.uid[0..1] == 'co') {
-                    list << collectionService.buildSummary(it.uid)
+                    list << collectionService.buildSummary(it)
                 } else if (pg[0..1] == 'in') {
-                    list << institutionService.buildSummary(it.uid)
+                    list << institutionService.buildSummary(it)
                 } else if (pg[0..1] == 'dr') {
-                    list << dataResourceService.buildSummary(it.uid)
+                    list << dataResourceService.buildSummary(it)
                 } else {
                     list << it.buildSummary()
                 }
@@ -234,11 +234,11 @@ class LookupController {
         }
         if (instance) {
             if (instance.uid[0..1] == 'co') {
-                render collectionService.buildSummary(instance.uid) as JSON
+                render collectionService.buildSummary(instance) as JSON
             } else if (pg[0..1] == 'in') {
-                render institutionService.buildSummary(instance.uid) as JSON
+                render institutionService.buildSummary(instance) as JSON
             } else if (pg[0..1] == 'dr') {
-                render dataResourceService.buildSummary(instance.uid) as JSON
+                render dataResourceService.buildSummary(instance) as JSON
             } else {
                 render instance.buildSummary() as JSON
             }
diff --git a/grails-app/controllers/au/org/ala/collectory/ReportsController.groovy b/grails-app/controllers/au/org/ala/collectory/ReportsController.groovy
index dda7501f..dcaa5ce3 100644
--- a/grails-app/controllers/au/org/ala/collectory/ReportsController.groovy
+++ b/grails-app/controllers/au/org/ala/collectory/ReportsController.groovy
@@ -86,7 +86,7 @@ class ReportsController {
 
     def codes = {
         [codeSummaries: (ProviderMap.list().collect {
-            collectionService.buildSummary(it.collection.uid)
+            collectionService.buildSummary(it.collection)
         }).sort {it.name}]
     }
 
@@ -218,13 +218,13 @@ class ReportsController {
     def attributions = {
         def collAttributions = []
         Collection.list([sort: 'name']).each {
-            ProviderGroupSummary pgs = collectionService.buildSummary(it.uid)
+            ProviderGroupSummary pgs = collectionService.buildSummary(it)
             List<Attribution> attribs = it.getAttributionList()
             def ats = new Attributions(pgs, attribs)
             collAttributions << ats
         }
         def instAttributions = Institution.list([sort: 'name']).collect {
-            ProviderGroupSummary pgs = institutionService.buildSummary(it.uid)
+            ProviderGroupSummary pgs = institutionService.buildSummary(it)
             List<Attribution> attribs = it.getAttributionList()
             new Attributions(pgs, attribs)
         }
@@ -249,7 +249,7 @@ class ReportsController {
                 }
                 // compare to num digistised
                 if (count == 0 || (count / it.numRecordsDigitised) < 0.7) {
-                    mrs << [collection: collectionService.buildSummary(it.uid), biocacheCount: count, claimed: it.numRecordsDigitised]
+                    mrs << [collection: collectionService.buildSummary(it), biocacheCount: count, claimed: it.numRecordsDigitised]
                 }
             }
         }

From d6f102d76bbb12e7268fd5e17c7843f684f5dc51 Mon Sep 17 00:00:00 2001
From: Adam Collins <adam.collins@csiro.au>
Date: Mon, 4 Dec 2023 19:50:44 +1000
Subject: [PATCH 08/13] fix service conflict

---
 .../CollectoryWebServicesInterceptor.groovy   |  4 -
 .../org/ala/collectory/DataController.groovy  | 76 +++++++++++++++----
 .../au/org/ala/collectory/UrlMappings.groovy  |  2 +
 3 files changed, 62 insertions(+), 20 deletions(-)

diff --git a/grails-app/controllers/au/org/ala/collectory/CollectoryWebServicesInterceptor.groovy b/grails-app/controllers/au/org/ala/collectory/CollectoryWebServicesInterceptor.groovy
index 10b32900..e00ec315 100644
--- a/grails-app/controllers/au/org/ala/collectory/CollectoryWebServicesInterceptor.groovy
+++ b/grails-app/controllers/au/org/ala/collectory/CollectoryWebServicesInterceptor.groovy
@@ -33,10 +33,6 @@ class CollectoryWebServicesInterceptor {
 
     }
     boolean before() {
-        // Failed to intercept this request in UrlMappings, doing it here and in `saveEntity`
-        if (request.method == "POST" && !params.uid) {
-            return true
-        }
 
         // set default role requirement for protected ROLE_EDITOR as the same info is only available to ROLE_EDITOR via the UI.
         String requiredRole = grailsApplication.config.ROLE_EDITOR
diff --git a/grails-app/controllers/au/org/ala/collectory/DataController.groovy b/grails-app/controllers/au/org/ala/collectory/DataController.groovy
index 2ac0ec5d..6074ba25 100644
--- a/grails-app/controllers/au/org/ala/collectory/DataController.groovy
+++ b/grails-app/controllers/au/org/ala/collectory/DataController.groovy
@@ -225,14 +225,12 @@ class DataController {
      * @param pg - optional instance specified by uid (added in beforeInterceptor)
      * @param json - the body of the request
      */
-    // NOTE  - Sine the same method below saveEntity is used for saving and updating an entity - OpenAPI specs for both save and update operations cannot be added on Code.
-    // API Gateway handles the below by using the {entity+} wildcard but for swagger specs in the documentation portal. additional spec has been added for just the /{entity} path - which allows Inserting of an entity
     @Operation(
             method = "POST",
             tags = "collection, institution, dataProvider, dataResource, tempDataResource, dataHub",
             operationId = "updateEntity",
-            summary = "Insert or  update an entity",
-            description = "Insert or update an  entity  - if uid is specified, entity must exist and is updated with the provided data",
+            summary = "Update an entity",
+            description = "Update an  entity  - if uid is specified, entity must exist and is updated with the provided data",
             parameters = [
                     @Parameter(
                             name = "entity",
@@ -245,10 +243,10 @@ class DataController {
                     @Parameter(
                             name = "uid",
                             in = PATH,
-                            description = "optional uid of an instance of entity",
+                            description = "uid of an instance of entity",
                             schema = @Schema(implementation = String),
                             example = "co43",
-                            required = false
+                            required = true
                     )
             ],
             requestBody = @RequestBody(
@@ -283,11 +281,6 @@ class DataController {
     @Path("/ws/{entity}/{uid}")
     @Produces("application/json")
     def saveEntity() {
-        // Failed to intercept this request in UrlMappings, doing it here and in `CollectoryWebServicesInterceptor`
-        if (!params.uid) {
-            getEntities()
-            return
-        }
 
         def ok = check(params)
         if (ok) {
@@ -322,6 +315,57 @@ class DataController {
         }
     }
 
+    @Operation(
+            method = "POST",
+            tags = "collection, institution, dataProvider, dataResource, tempDataResource, dataHub",
+            operationId = "createEntity",
+            summary = "create entity",
+            description = "create entity",
+            parameters = [
+                    @Parameter(
+                            name = "entity",
+                            in = PATH,
+                            description = "entity i.e.  collection, institution, dataProvider, dataResource, tempDataResource, dataHub",
+                            schema = @Schema(implementation = String),
+                            example = "collection",
+                            required = true
+                    )
+            ],
+            requestBody = @RequestBody(
+                    required = true,
+                    description = "A JSON object containing the entity to be saved or updated",
+                    content = [
+                            @Content(
+                                    mediaType = "application/json",
+                                    schema = @Schema(implementation = Object)
+                            )
+                    ]
+            ),
+            responses = [
+                    @ApiResponse(
+                            description = "Status of operation",
+                            responseCode = "201",
+                            content = [
+                                    @Content(
+                                            mediaType = "application/json",
+                                            schema = @Schema(implementation = Object)
+                                    )
+                            ],
+                            headers = [
+                                    @Header(name = 'Access-Control-Allow-Headers', description = "CORS header", schema = @Schema(type = "String")),
+                                    @Header(name = 'Access-Control-Allow-Methods', description = "CORS header", schema = @Schema(type = "String")),
+                                    @Header(name = 'Access-Control-Allow-Origin', description = "CORS header", schema = @Schema(type = "String"))
+                            ]
+                    )
+            ],
+            security = [@SecurityRequirement(name = 'openIdConnect')]
+    )
+    @Path("/ws/{entity}")
+    @Produces("application/json")
+    def createEntity() {
+        saveEntity()
+    }
+
     /**
      * Define some variations on the level of detail returned for lists.
      */
@@ -496,9 +540,9 @@ class DataController {
     @Operation(
             method = "POST",
             tags = "collection, institution, dataProvider, dataResource, tempDataResource, dataHub",
-            operationId = "getEntities",
-            summary = "Get an entity for a list of entity uids",
-            description = "Get detailed information for a list of entities",
+            operationId = "findEntities",
+            summary = "Find entities for a list of entity uids",
+            description = "Find detailed information for a list of entities",
             parameters = [
                     @Parameter(
                             name = "entity",
@@ -541,9 +585,9 @@ class DataController {
             ],
             security = []
     )
-    @Path("/ws/{entity}")
+    @Path("/ws/find/{entity}")
     @Produces("application/json")
-    def getEntities() {
+    def findEntities() {
 
         def result = []
 
diff --git a/grails-app/controllers/au/org/ala/collectory/UrlMappings.groovy b/grails-app/controllers/au/org/ala/collectory/UrlMappings.groovy
index 3678512b..52cca441 100644
--- a/grails-app/controllers/au/org/ala/collectory/UrlMappings.groovy
+++ b/grails-app/controllers/au/org/ala/collectory/UrlMappings.groovy
@@ -60,6 +60,8 @@ class UrlMappings {
         // data sets
         "/datasets"(controller: 'public', action: 'dataSets')
 
+        "/ws/find/$entity(.$format)?"(controller: 'data', action: 'findEntities')
+
         // data services
         "/ws/$entity/count/$groupBy?"(controller: 'data', action: 'count') {
             constraints {

From 064a5044b7bc0a5b256b81e90b2307c5cb8f2c76 Mon Sep 17 00:00:00 2001
From: Adam Collins <adam.collins@csiro.au>
Date: Tue, 5 Dec 2023 13:47:25 +1000
Subject: [PATCH 09/13] fix pageOptionsPopup

---
 .../taglib/au/org/ala/collectory/CollectoryTagLib.groovy  | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/grails-app/taglib/au/org/ala/collectory/CollectoryTagLib.groovy b/grails-app/taglib/au/org/ala/collectory/CollectoryTagLib.groovy
index cdfb3ed5..7dee5244 100644
--- a/grails-app/taglib/au/org/ala/collectory/CollectoryTagLib.groovy
+++ b/grails-app/taglib/au/org/ala/collectory/CollectoryTagLib.groovy
@@ -1455,14 +1455,14 @@ class CollectoryTagLib {
         }
         if (providers) {
             boolean first = true
-            providers.each {
+            providers.each { provider ->
                 // only write the header if we have at least one resource
                 if (first) {
                     out << "<p class='viewList' style='margin-top:5px;'>View data sources<br/><ul>\n"
                     first = false
                 }
                 out << "<li>" +
-                        link(action:'show',id:it.uid) {it.name} +
+                        link(action:'show',id:provider.uid) {provider.name} +
                         "</li>\n"
             }
             out << "</ul></p>\n"
@@ -1476,14 +1476,14 @@ class CollectoryTagLib {
         }
         if (consumers) {
             boolean first = true
-            consumers.each {
+            consumers.each { consumer ->
                 // only write the header if we have at least one resource
                 if (first) {
                     out << "<p class='viewList' style='margin-top:5px;'>View data consumers<br/><ul>\n"
                     first = false
                 }
                 out << "<li>" +
-                        link(action:'show',id:it.uid) {it.name} +
+                        link(action:'show',id:consumer.uid) {consumer.name} +
                         "</li>\n"
             }
             out << "</ul></p>\n"

From 5ca06fe940bda4100a9a4ad177624540a9603d5e Mon Sep 17 00:00:00 2001
From: Adam Collins <adam.collins@csiro.au>
Date: Tue, 12 Dec 2023 13:45:26 +1000
Subject: [PATCH 10/13] make gbifCountryToAttribute mandatory

---
 grails-app/conf/application.yml               |  5 ++
 .../collectory/DataProviderController.groovy  |  5 +-
 .../collectory/ProviderGroupController.groovy |  4 +-
 .../au/org/ala/collectory/DataProvider.groovy |  2 +-
 .../au/org/ala/collectory/Institution.groovy  |  2 +-
 grails-app/i18n/messages.properties           |  1 +
 grails-app/i18n/messages_ca.properties        |  3 +-
 grails-app/i18n/messages_da.properties        |  3 +-
 grails-app/i18n/messages_de.properties        |  3 +-
 grails-app/i18n/messages_de_AT.properties     |  3 +-
 grails-app/i18n/messages_de_LU.properties     |  3 +-
 grails-app/i18n/messages_es.properties        |  3 +-
 grails-app/i18n/messages_eu.properties        |  3 +-
 grails-app/i18n/messages_fr.properties        |  3 +-
 grails-app/i18n/messages_it.properties        |  3 +-
 grails-app/i18n/messages_ja.properties        |  3 +-
 grails-app/i18n/messages_nl.properties        |  3 +-
 grails-app/i18n/messages_pt_BR.properties     |  3 +-
 grails-app/i18n/messages_pt_PT.properties     |  3 +-
 grails-app/i18n/messages_ru.properties        |  3 +-
 grails-app/i18n/messages_sv.properties        |  3 +-
 grails-app/i18n/messages_th.properties        | 59 ++++++++++---------
 grails-app/i18n/messages_zh.properties        |  3 +-
 grails-app/i18n/messages_zh_CN.properties     |  3 +-
 grails-app/init/collectory/BootStrap.groovy   | 11 +++-
 grails-app/migrations/5.1.0.2.sql             |  6 ++
 grails-app/migrations/changelog.xml           |  4 ++
 .../au/org/ala/collectory/CrudService.groovy  |  4 +-
 .../ala/collectory/DataLoaderService.groovy   |  9 +--
 .../ala/collectory/GbifRegistryService.groovy |  8 ++-
 grails-app/views/dataProvider/list.gsp        |  4 ++
 grails-app/views/shared/_institutionList.gsp  |  3 +
 grails-app/views/shared/base.gsp              |  6 +-
 grails-app/views/shared/gbif.gsp              |  6 +-
 34 files changed, 119 insertions(+), 71 deletions(-)
 create mode 100644 grails-app/migrations/5.1.0.2.sql

diff --git a/grails-app/conf/application.yml b/grails-app/conf/application.yml
index 6f0f28c3..95adee0b 100644
--- a/grails-app/conf/application.yml
+++ b/grails-app/conf/application.yml
@@ -112,6 +112,9 @@ uploadExternalUrlPath: http://localhost/upload
 gbifApiUrl: 'https://api.gbif.org/v1/'
 gbifUniqueKeyTerm: "http://rs.gbif.org/terms/1.0/gbifID"
 
+# country 3 digit ISO code. Non-empty value required at startup
+gbifDefaultEntityCountry:
+
 gbif:
     citations:
         lookup: "https://www.gbif.org/api/resource/search?contentType=literature&gbifDatasetKey="
@@ -404,6 +407,7 @@ environments:
             username: specieslist
             password: "specieslist"
             logSql: false
+        gbifDefaultEntityCountry: AUS
     test:
         grails:
             assets:
@@ -414,6 +418,7 @@ environments:
         dataSource:
             dbCreate: update
             url: jdbc:h2:mem:testDb;MVCC=TRUE;LOCK_TIMEOUT=10000;DB_CLOSE_ON_EXIT=FALSE
+        gbifDefaultEntityCountry: AUS
     production:
         grails:
             assets:
diff --git a/grails-app/controllers/au/org/ala/collectory/DataProviderController.groovy b/grails-app/controllers/au/org/ala/collectory/DataProviderController.groovy
index b91c74a2..1f871711 100644
--- a/grails-app/controllers/au/org/ala/collectory/DataProviderController.groovy
+++ b/grails-app/controllers/au/org/ala/collectory/DataProviderController.groovy
@@ -21,7 +21,6 @@ class DataProviderController extends ProviderGroupController {
         }
         params.max = Math.min(params.max ? params.int('max') : 10000, 10000)
         params.sort = params.sort ?: "name"
-        params.fetch = [resources: 'join']
         activityLogService.log username(), isAdmin(), Action.LIST
         [instanceList: DataProvider.list(params), entityType: 'DataProvider', instanceTotal: DataProvider.count()]
     }
@@ -97,7 +96,7 @@ class DataProviderController extends ProviderGroupController {
         def organizationKey = params.organizationKey
         log.info "Importing organization "+organizationKey+" as data provider"
 
-        DataProvider dp = new DataProvider(uid: idGeneratorService.getNextDataProviderId(), userLastModified: collectoryAuthService?.username())
+        DataProvider dp = new DataProvider(uid: idGeneratorService.getNextDataProviderId(), userLastModified: collectoryAuthService?.username(), gbifCountryToAttribute: grailsApplication.config.gbifDefaultEntityCountry)
         gbifRegistryService.populateDataProviderFromOrganization(dp, organizationKey)
 
         if (!dp.hasErrors()) {
@@ -131,7 +130,7 @@ class DataProviderController extends ProviderGroupController {
         organizations.each { organization ->
             def dp = DataProvider.findByGbifRegistryKey(organization.key)
             if (!dp) {
-                dp = new DataProvider(uid: idGeneratorService.getNextDataProviderId(), userLastModified: collectoryAuthService?.username())
+                dp = new DataProvider(uid: idGeneratorService.getNextDataProviderId(), userLastModified: collectoryAuthService?.username(), gbifCountryToAttribute: grailsApplication.config.gbifDefaultEntityCountry)
                 gbifRegistryService.populateDataProviderFromOrganization(dp, organization.key)
 
                 if (!dp.hasErrors()) {
diff --git a/grails-app/controllers/au/org/ala/collectory/ProviderGroupController.groovy b/grails-app/controllers/au/org/ala/collectory/ProviderGroupController.groovy
index 691bdf3f..5ba2cbe0 100644
--- a/grails-app/controllers/au/org/ala/collectory/ProviderGroupController.groovy
+++ b/grails-app/controllers/au/org/ala/collectory/ProviderGroupController.groovy
@@ -152,9 +152,9 @@ abstract class ProviderGroupController {
                 }
                 break
             case Institution.ENTITY_TYPE:
-                pg = new Institution(uid: idGeneratorService.getNextInstitutionId(), name: name, userLastModified: collectoryAuthService?.username()); break
+                pg = new Institution(uid: idGeneratorService.getNextInstitutionId(), name: name, userLastModified: collectoryAuthService?.username(), gbifCountryToAttribute: grailsApplication.config.gbifDefaultEntityCountry); break
             case DataProvider.ENTITY_TYPE:
-                pg = new DataProvider(uid: idGeneratorService.getNextDataProviderId(), name: name, userLastModified: collectoryAuthService?.username()); break
+                pg = new DataProvider(uid: idGeneratorService.getNextDataProviderId(), name: name, userLastModified: collectoryAuthService?.username(), gbifCountryToAttribute: grailsApplication.config.gbifDefaultEntityCountry); break
             case DataResource.ENTITY_TYPE:
                 pg = new DataResource(uid: idGeneratorService.getNextDataResourceId(), name: name, userLastModified: collectoryAuthService?.username())
                 if (params.dataProviderUid && DataProvider.findByUid(params.dataProviderUid)) {
diff --git a/grails-app/domain/au/org/ala/collectory/DataProvider.groovy b/grails-app/domain/au/org/ala/collectory/DataProvider.groovy
index 14772d72..51521ade 100644
--- a/grails-app/domain/au/org/ala/collectory/DataProvider.groovy
+++ b/grails-app/domain/au/org/ala/collectory/DataProvider.groovy
@@ -89,7 +89,7 @@ class DataProvider implements ProviderGroup, Serializable {
         gbifRegistryKey(nullable:true, maxSize:36)
         hiddenJSON(nullable:true, blank: false)
         keywords(nullable:true)
-        gbifCountryToAttribute(nullable:true, maxSize: 3)
+        gbifCountryToAttribute(nullable:false, maxSize: 3)
     }
 
     /**
diff --git a/grails-app/domain/au/org/ala/collectory/Institution.groovy b/grails-app/domain/au/org/ala/collectory/Institution.groovy
index 859fdb44..d556c1f8 100644
--- a/grails-app/domain/au/org/ala/collectory/Institution.groovy
+++ b/grails-app/domain/au/org/ala/collectory/Institution.groovy
@@ -47,7 +47,7 @@ class Institution implements ProviderGroup, Serializable {
         institutionType(nullable:true, maxSize:45)
         collections(nullable:true)
         childInstitutions(nullable:true)
-        gbifCountryToAttribute(nullable:true, maxSize: 3)
+        gbifCountryToAttribute(nullable:false, maxSize: 3)
     }
 
     static transients = ['summary','mappable']
diff --git a/grails-app/i18n/messages.properties b/grails-app/i18n/messages.properties
index 8aaf0d10..40e458e2 100644
--- a/grails-app/i18n/messages.properties
+++ b/grails-app/i18n/messages.properties
@@ -2315,3 +2315,4 @@ resourceType.annotations.description=This dataset references data from other dat
 resourceType.records.description=This dataset that contains primary occurrence data for species.
 resourceType.events.description=This dataset that contains event based data for species which may include sampling information
 resourceType.species-list.description=This dataset contains a list of species with an associated trait.
+gbifCountryToAttribute.label=GBIF Country To Attribute
diff --git a/grails-app/i18n/messages_ca.properties b/grails-app/i18n/messages_ca.properties
index 226ccb2a..0c25eca8 100644
--- a/grails-app/i18n/messages_ca.properties
+++ b/grails-app/i18n/messages_ca.properties
@@ -2032,4 +2032,5 @@ decimalLatLongConverionFailed=S'ha produ\u00eft un error en la conversi\u00f3 de
 decimalLatLongCalculatedFromGridReference=Latitud Longitud calculades a partir d'una malla de refer\u00e8ncia
 nameNotSupplied=Nom no proporcionat
 zeroLatitude=Latitud zero
-zeroLongitude=Longitud zero
\ No newline at end of file
+zeroLongitude=Longitud zero
+gbifCountryToAttribute.label=GBIF Country To Attribute
diff --git a/grails-app/i18n/messages_da.properties b/grails-app/i18n/messages_da.properties
index 58674d2b..e80a26f3 100644
--- a/grails-app/i18n/messages_da.properties
+++ b/grails-app/i18n/messages_da.properties
@@ -185,6 +185,7 @@ typeMismatch.java.lang.Long=Feltet {0} skal v\u00e6re et valid tal
 typeMismatch.java.lang.Short=Feltet {0} skal v\u00e6re et valid tal
 typeMismatch.java.math.BigDecimal=Feltet {0} skal v\u00e6re et valid tal
 typeMismatch.java.math.BigInteger=Feltet {0} skal v\u00e6re et valid tal
+gbifCountryToAttribute.label=GBIF Country To Attribute
 
 # tool tips
 
@@ -221,4 +222,4 @@ typeMismatch.java.math.BigInteger=Feltet {0} skal v\u00e6re et valid tal
 
 
 
-#issue types
\ No newline at end of file
+#issue types
diff --git a/grails-app/i18n/messages_de.properties b/grails-app/i18n/messages_de.properties
index ce5b8861..2fa46c47 100644
--- a/grails-app/i18n/messages_de.properties
+++ b/grails-app/i18n/messages_de.properties
@@ -2031,4 +2031,5 @@ decimalLatLongConverionFailed=Dezimal-Breiten/L\u00e4ngen Umwandlung fehlgeschla
 decimalLatLongCalculatedFromGridReference=Breitengrad-L\u00e4nge berechnet aus Rasterreferenz
 nameNotSupplied=Name nicht angegeben
 zeroLatitude=Null Breitengrad
-zeroLongitude=Null L\u00e4ngengrad
\ No newline at end of file
+zeroLongitude=Null L\u00e4ngengrad
+gbifCountryToAttribute.label=GBIF Country To Attribute
diff --git a/grails-app/i18n/messages_de_AT.properties b/grails-app/i18n/messages_de_AT.properties
index 227f4aec..f3e319a9 100644
--- a/grails-app/i18n/messages_de_AT.properties
+++ b/grails-app/i18n/messages_de_AT.properties
@@ -2031,4 +2031,5 @@ decimalLatLongConverionFailed=Dezimal-Breiten/L\u00e4ngen Umwandlung fehlgeschla
 decimalLatLongCalculatedFromGridReference=Breitengrad-L\u00e4nge berechnet aus Rasterreferenz
 nameNotSupplied=Name nicht angegeben
 zeroLatitude=Null Breitengrad
-zeroLongitude=Null L\u00e4ngengrad
\ No newline at end of file
+zeroLongitude=Null L\u00e4ngengrad
+gbifCountryToAttribute.label=GBIF Country To Attribute
diff --git a/grails-app/i18n/messages_de_LU.properties b/grails-app/i18n/messages_de_LU.properties
index c123dc9f..30c69884 100644
--- a/grails-app/i18n/messages_de_LU.properties
+++ b/grails-app/i18n/messages_de_LU.properties
@@ -2031,4 +2031,5 @@ decimalLatLongConverionFailed=Dezimal-Breiten/L\u00e4ngen Umwandlung fehlgeschla
 decimalLatLongCalculatedFromGridReference=Breitengrad-L\u00e4nge berechnet aus Rasterreferenz
 nameNotSupplied=Name nicht angegeben
 zeroLatitude=Null Breitengrad
-zeroLongitude=Null L\u00e4ngengrad
\ No newline at end of file
+zeroLongitude=Null L\u00e4ngengrad
+gbifCountryToAttribute.label=GBIF Country To Attribute
diff --git a/grails-app/i18n/messages_es.properties b/grails-app/i18n/messages_es.properties
index d3bfdf1b..368261ba 100644
--- a/grails-app/i18n/messages_es.properties
+++ b/grails-app/i18n/messages_es.properties
@@ -2032,4 +2032,5 @@ decimalLatLongConverionFailed=La conversi\u00f3n decimal latitud/longitud ha fal
 decimalLatLongCalculatedFromGridReference=Latitud Longitud calculadas a partir de una malla de referencia
 nameNotSupplied=Nombre no proporcionado
 zeroLatitude=Latitud cero
-zeroLongitude=Longitud cero
\ No newline at end of file
+zeroLongitude=Longitud cero
+gbifCountryToAttribute.label=GBIF Country To Attribute
diff --git a/grails-app/i18n/messages_eu.properties b/grails-app/i18n/messages_eu.properties
index 7a98253f..e1046637 100644
--- a/grails-app/i18n/messages_eu.properties
+++ b/grails-app/i18n/messages_eu.properties
@@ -878,6 +878,7 @@ dataresource.show.dc=Datuak onartzea
 
 manage.show.temp.startdate=Hasiera data
 manage.show.temp.enddate=Amaiera data
+gbifCountryToAttribute.label=GBIF Country To Attribute
 
 
 
@@ -950,4 +951,4 @@ manage.show.temp.enddate=Amaiera data
 
 
 
-#issue types
\ No newline at end of file
+#issue types
diff --git a/grails-app/i18n/messages_fr.properties b/grails-app/i18n/messages_fr.properties
index 9e0445cb..f8774f01 100644
--- a/grails-app/i18n/messages_fr.properties
+++ b/grails-app/i18n/messages_fr.properties
@@ -2032,4 +2032,5 @@ decimalLatLongConverionFailed=\u00c9chec de la conversion entre degr\u00e9s d\u0
 decimalLatLongCalculatedFromGridReference=Latitude et longitude calcul\u00e9es \u00e0 partir de la grille de r\u00e9f\u00e9rence
 nameNotSupplied=Nom non fourni
 zeroLatitude=Z\u00e9ro de latitude
-zeroLongitude=Z\u00e9ro de longitude
\ No newline at end of file
+zeroLongitude=Z\u00e9ro de longitude
+gbifCountryToAttribute.label=GBIF Country To Attribute
diff --git a/grails-app/i18n/messages_it.properties b/grails-app/i18n/messages_it.properties
index 24aa37a8..c68718d1 100644
--- a/grails-app/i18n/messages_it.properties
+++ b/grails-app/i18n/messages_it.properties
@@ -148,6 +148,7 @@ default.not.unique.message=La propriet\u00e0 [{0}] della classe [{1}] con valore
 
 default.paginate.prev=Precedente
 default.paginate.next=Successivo
+gbifCountryToAttribute.label=GBIF Country To Attribute
 
 
 
@@ -192,4 +193,4 @@ default.paginate.next=Successivo
 
 
 
-#issue types
\ No newline at end of file
+#issue types
diff --git a/grails-app/i18n/messages_ja.properties b/grails-app/i18n/messages_ja.properties
index d0ae41a9..f0fbcb22 100644
--- a/grails-app/i18n/messages_ja.properties
+++ b/grails-app/i18n/messages_ja.properties
@@ -148,6 +148,7 @@ default.not.unique.message=\u30af\u30e9\u30b9[{1}]\u30d7\u30ed\u30d1\u30c6\u30a3
 
 default.paginate.prev=\u623b\u308b
 default.paginate.next=\u6b21\u3078
+gbifCountryToAttribute.label=GBIF Country To Attribute
 
 
 
@@ -192,4 +193,4 @@ default.paginate.next=\u6b21\u3078
 
 
 
-#issue types
\ No newline at end of file
+#issue types
diff --git a/grails-app/i18n/messages_nl.properties b/grails-app/i18n/messages_nl.properties
index f9a65ac8..28d9ea74 100644
--- a/grails-app/i18n/messages_nl.properties
+++ b/grails-app/i18n/messages_nl.properties
@@ -186,6 +186,7 @@ typeMismatch.java.lang.Long=Attribuut {0} is geen geldig nummer
 typeMismatch.java.lang.Short=Attribuut {0} is geen geldig nummer
 typeMismatch.java.math.BigDecimal=Attribuut {0} is geen geldig nummer
 typeMismatch.java.math.BigInteger=Attribuut {0} is geen geldig nummer
+gbifCountryToAttribute.label=GBIF Country To Attribute
 
 # tool tips
 
@@ -222,4 +223,4 @@ typeMismatch.java.math.BigInteger=Attribuut {0} is geen geldig nummer
 
 
 
-#issue types
\ No newline at end of file
+#issue types
diff --git a/grails-app/i18n/messages_pt_BR.properties b/grails-app/i18n/messages_pt_BR.properties
index 8acfd49c..c88f7f03 100644
--- a/grails-app/i18n/messages_pt_BR.properties
+++ b/grails-app/i18n/messages_pt_BR.properties
@@ -2029,4 +2029,5 @@ decimalLatLongConverionFailed=Falha na convers\u00e3o decimal de latitude/longit
 decimalLatLongCalculatedFromGridReference=Latitude e longitude calculado a partir da refer\u00eancia de grade
 nameNotSupplied=Nome n\u00e3o fornecido
 zeroLatitude=Latitude zero
-zeroLongitude=Longitude zero
\ No newline at end of file
+zeroLongitude=Longitude zero
+gbifCountryToAttribute.label=GBIF Country To Attribute
diff --git a/grails-app/i18n/messages_pt_PT.properties b/grails-app/i18n/messages_pt_PT.properties
index 821c804a..f1612d80 100644
--- a/grails-app/i18n/messages_pt_PT.properties
+++ b/grails-app/i18n/messages_pt_PT.properties
@@ -1665,6 +1665,7 @@ datasets.js.facets07=Os tipos de informa\u00e7\u00f5es dispon\u00edveis no conju
 datasets.js.facets08=Cont\u00e9m
 datasets.js.facets09=Institui\u00e7\u00e3o
 datasets.js.facets10=A organiza\u00e7\u00e3o ou institui\u00e7\u00e3o que \u00e9 a fonte ou o deposit\u00e1rio do conjunto de dados
+gbifCountryToAttribute.label=GBIF Country To Attribute
 
 
 
@@ -1672,4 +1673,4 @@ datasets.js.facets10=A organiza\u00e7\u00e3o ou institui\u00e7\u00e3o que \u00e9
 
 
 
-#issue types
\ No newline at end of file
+#issue types
diff --git a/grails-app/i18n/messages_ru.properties b/grails-app/i18n/messages_ru.properties
index c4153063..84830a0c 100644
--- a/grails-app/i18n/messages_ru.properties
+++ b/grails-app/i18n/messages_ru.properties
@@ -165,6 +165,7 @@ typeMismatch.java.lang.Long=\u0417\u043d\u0430\u0447\u0435\u043d\u0438\u0435 \u0
 typeMismatch.java.lang.Short=\u0417\u043d\u0430\u0447\u0435\u043d\u0438\u0435 \u043f\u043e\u043b\u044f {0} \u043d\u0435 \u044f\u0432\u043b\u044f\u0435\u0442\u0441\u044f \u0434\u043e\u043f\u0443\u0441\u0442\u0438\u043c\u044b\u043c \u0447\u0438\u0441\u043b\u043e\u043c
 typeMismatch.java.math.BigDecimal=\u0417\u043d\u0430\u0447\u0435\u043d\u0438\u0435 \u043f\u043e\u043b\u044f {0} \u043d\u0435 \u044f\u0432\u043b\u044f\u0435\u0442\u0441\u044f \u0434\u043e\u043f\u0443\u0441\u0442\u0438\u043c\u044b\u043c \u0447\u0438\u0441\u043b\u043e\u043c
 typeMismatch.java.math.BigInteger=\u0417\u043d\u0430\u0447\u0435\u043d\u0438\u0435 \u043f\u043e\u043b\u044f {0} \u043d\u0435 \u044f\u0432\u043b\u044f\u0435\u0442\u0441\u044f \u0434\u043e\u043f\u0443\u0441\u0442\u0438\u043c\u044b\u043c \u0447\u0438\u0441\u043b\u043e\u043c
+gbifCountryToAttribute.label=GBIF Country To Attribute
 
 # tool tips
 
@@ -201,4 +202,4 @@ typeMismatch.java.math.BigInteger=\u0417\u043d\u0430\u0447\u0435\u043d\u0438\u04
 
 
 
-#issue types
\ No newline at end of file
+#issue types
diff --git a/grails-app/i18n/messages_sv.properties b/grails-app/i18n/messages_sv.properties
index 9254d143..a306ea98 100644
--- a/grails-app/i18n/messages_sv.properties
+++ b/grails-app/i18n/messages_sv.properties
@@ -641,6 +641,7 @@ datasets.js.displaytext03=CC BY-SA
 datasets.js.displaytext04=CC BY-NC-SA
 datasets.js.facets08=Inneh\u00e5ller
 datasets.js.facets09=Organisation
+gbifCountryToAttribute.label=GBIF Country To Attribute
 
 
 
@@ -648,4 +649,4 @@ datasets.js.facets09=Organisation
 
 
 
-#issue types
\ No newline at end of file
+#issue types
diff --git a/grails-app/i18n/messages_th.properties b/grails-app/i18n/messages_th.properties
index aadc4125..0317c9e6 100644
--- a/grails-app/i18n/messages_th.properties
+++ b/grails-app/i18n/messages_th.properties
@@ -1,35 +1,36 @@
-default.doesnt.match.message=คุณสมบัติ [{0}] ของคลาส [{1}] ซึ่งมีค่าเป็น [{2}] ไม่ถูกต้องตามรูปแบบที่กำหนดไว้ใน [{3}]
-default.invalid.url.message=คุณสมบัติ [{0}] ของคลาส [{1}] ซึ่งมีค่าเป็น [{2}] ไม่ถูกต้องตามรูปแบบ URL
-default.invalid.creditCard.message=คุณสมบัติ [{0}] ของคลาส [{1}] ซึ่งมีค่าเป็น [{2}] ไม่ถูกต้องตามรูปแบบหมายเลขบัตรเครดิต
-default.invalid.email.message=คุณสมบัติ [{0}] ของคลาส [{1}] ซึ่งมีค่าเป็น [{2}] ไม่ถูกต้องตามรูปแบบอีเมล์
-default.invalid.range.message=คุณสมบัติ [{0}] ของคลาส [{1}] ซึ่งมีค่าเป็น [{2}] ไม่ได้มีค่าที่ถูกต้องในช่วงจาก [{3}] ถึง [{4}]
-default.invalid.size.message=คุณสมบัติ [{0}] ของคลาส [{1}] ซึ่งมีค่าเป็น [{2}] ไม่ได้มีขนาดที่ถูกต้องในช่วงจาก [{3}] ถึง [{4}]
-default.invalid.max.message=คุณสมบัติ [{0}] ของคลาส [{1}] ซึ่งมีค่าเป็น [{2}] มีค่าเกิดกว่าค่ามากสุด [{3}]
-default.invalid.min.message=คุณสมบัติ [{0}] ของคลาส [{1}] ซึ่งมีค่าเป็น [{2}] มีค่าน้อยกว่าค่าต่ำสุด  [{3}]
-default.invalid.max.size.message=คุณสมบัติ [{0}] ของคลาส [{1}] ซึ่งมีค่าเป็น [{2}] มีขนาดเกินกว่าขนาดมากสุดของ [{3}]
-default.invalid.min.size.message=คุณสมบัติ [{0}] ของคลาส [{1}] ซึ่งมีค่าเป็น [{2}] มีขนาดต่ำกว่าขนาดต่ำสุดของ  [{3}]
-default.invalid.validator.message=คุณสมบัติ [{0}] ของคลาส [{1}] ซึ่งมีค่าเป็น [{2}] ไม่ผ่านการทวนสอบค่าที่ตั้งขึ้น
-default.not.inlist.message=คุณสมบัติ [{0}] ของคลาส [{1}] ซึ่งมีค่าเป็น [{2}] ไม่ได้อยู่ในรายการต่อไปนี้  [{3}]
-default.blank.message=คุณสมบัติ [{0}] ของคลาส [{1}] ไม่สามารถเป็นค่าว่างได้
-default.not.equal.message=คุณสมบัติ [{0}] ของคลาส [{1}] ซึ่งมีค่าเป็น [{2}] ไม่สามารถเท่ากับ [{3}] ได้
-default.null.message=คุณสมบัติ [{0}] ของคลาส [{1}] ไม่สามารถเป็น null ได้
-default.not.unique.message=คุณสมบัติ [{0}] ของคลาส [{1}] ซึ่งมีค่าเป็น [{2}] จะต้องไม่ซ้ำ (unique)
+default.doesnt.match.message=คุณสมบัติ [{0}] ของคลาส [{1}] ซึ่งมีค่าเป็น [{2}] ไม่ถูกต้องตามรูปแบบที่กำหนดไว้ใน [{3}]
+default.invalid.url.message=คุณสมบัติ [{0}] ของคลาส [{1}] ซึ่งมีค่าเป็น [{2}] ไม่ถูกต้องตามรูปแบบ URL
+default.invalid.creditCard.message=คุณสมบัติ [{0}] ของคลาส [{1}] ซึ่งมีค่าเป็น [{2}] ไม่ถูกต้องตามรูปแบบหมายเลขบัตรเครดิต
+default.invalid.email.message=คุณสมบัติ [{0}] ของคลาส [{1}] ซึ่งมีค่าเป็น [{2}] ไม่ถูกต้องตามรูปแบบอีเมล์
+default.invalid.range.message=คุณสมบัติ [{0}] ของคลาส [{1}] ซึ่งมีค่าเป็น [{2}] ไม่ได้มีค่าที่ถูกต้องในช่วงจาก [{3}] ถึง [{4}]
+default.invalid.size.message=คุณสมบัติ [{0}] ของคลาส [{1}] ซึ่งมีค่าเป็น [{2}] ไม่ได้มีขนาดที่ถูกต้องในช่วงจาก [{3}] ถึง [{4}]
+default.invalid.max.message=คุณสมบัติ [{0}] ของคลาส [{1}] ซึ่งมีค่าเป็น [{2}] มีค่าเกิดกว่าค่ามากสุด [{3}]
+default.invalid.min.message=คุณสมบัติ [{0}] ของคลาส [{1}] ซึ่งมีค่าเป็น [{2}] มีค่าน้อยกว่าค่าต่ำสุด  [{3}]
+default.invalid.max.size.message=คุณสมบัติ [{0}] ของคลาส [{1}] ซึ่งมีค่าเป็น [{2}] มีขนาดเกินกว่าขนาดมากสุดของ [{3}]
+default.invalid.min.size.message=คุณสมบัติ [{0}] ของคลาส [{1}] ซึ่งมีค่าเป็น [{2}] มีขนาดต่ำกว่าขนาดต่ำสุดของ  [{3}]
+default.invalid.validator.message=คุณสมบัติ [{0}] ของคลาส [{1}] ซึ่งมีค่าเป็น [{2}] ไม่ผ่านการทวนสอบค่าที่ตั้งขึ้น
+default.not.inlist.message=คุณสมบัติ [{0}] ของคลาส [{1}] ซึ่งมีค่าเป็น [{2}] ไม่ได้อยู่ในรายการต่อไปนี้  [{3}]
+default.blank.message=คุณสมบัติ [{0}] ของคลาส [{1}] ไม่สามารถเป็นค่าว่างได้
+default.not.equal.message=คุณสมบัติ [{0}] ของคลาส [{1}] ซึ่งมีค่าเป็น [{2}] ไม่สามารถเท่ากับ [{3}] ได้
+default.null.message=คุณสมบัติ [{0}] ของคลาส [{1}] ไม่สามารถเป็น null ได้
+default.not.unique.message=คุณสมบัติ [{0}] ของคลาส [{1}] ซึ่งมีค่าเป็น [{2}] จะต้องไม่ซ้ำ (unique)
 
-default.paginate.prev=ก่อนหน้า
-default.paginate.next=ถัดไป
+default.paginate.prev=ก่อนหน้า
+default.paginate.next=ถัดไป
 
-default.boolean.true=จริง
-default.boolean.false=เท็จ
+default.boolean.true=จริง
+default.boolean.false=เท็จ
 default.date.format=dd-MM-yyyy HH:mm:ss z
 default.number.format=0
 
 # Data binding errors. Use "typeMismatch.$className.$propertyName to customize (eg typeMismatch.Book.author)
-typeMismatch.java.net.URL=คุณสมบัติ '{0}' จะต้องเป็นค่า URL ที่ถูกต้อง
-typeMismatch.java.net.URI=คุณสมบัติ '{0}' จะต้องเป็นค่า URI ที่ถูกต้อง
-typeMismatch.java.util.Date=คุณสมบัติ '{0}' จะต้องมีค่าเป็นวันที่
-typeMismatch.java.lang.Double=คุณสมบัติ '{0}' จะต้องมีค่าเป็นจำนวนประเภท Double
-typeMismatch.java.lang.Integer=คุณสมบัติ '{0}' จะต้องมีค่าเป็นจำนวนประเภท Integer
-typeMismatch.java.lang.Long=คุณสมบัติ '{0}' จะต้องมีค่าเป็นจำนวนประเภท Long
-typeMismatch.java.lang.Short=คุณสมบัติ '{0}' จะต้องมีค่าเป็นจำนวนประเภท Short
-typeMismatch.java.math.BigDecimal=คุณสมบัติ '{0}' จะต้องมีค่าเป็นจำนวนประเภท BigDecimal
-typeMismatch.java.math.BigInteger=คุณสมบัติ '{0}' จะต้องมีค่าเป็นจำนวนประเภท BigInteger
\ No newline at end of file
+typeMismatch.java.net.URL=คุณสมบัติ '{0}' จะต้องเป็นค่า URL ที่ถูกต้อง
+typeMismatch.java.net.URI=คุณสมบัติ '{0}' จะต้องเป็นค่า URI ที่ถูกต้อง
+typeMismatch.java.util.Date=คุณสมบัติ '{0}' จะต้องมีค่าเป็นวันที่
+typeMismatch.java.lang.Double=คุณสมบัติ '{0}' จะต้องมีค่าเป็นจำนวนประเภท Double
+typeMismatch.java.lang.Integer=คุณสมบัติ '{0}' จะต้องมีค่าเป็นจำนวนประเภท Integer
+typeMismatch.java.lang.Long=คุณสมบัติ '{0}' จะต้องมีค่าเป็นจำนวนประเภท Long
+typeMismatch.java.lang.Short=คุณสมบัติ '{0}' จะต้องมีค่าเป็นจำนวนประเภท Short
+typeMismatch.java.math.BigDecimal=คุณสมบัติ '{0}' จะต้องมีค่าเป็นจำนวนประเภท BigDecimal
+typeMismatch.java.math.BigInteger=คุณสมบัติ '{0}' จะต้องมีค่าเป็นจำนวนประเภท BigInteger
+gbifCountryToAttribute.label=GBIF Country To Attribute
diff --git a/grails-app/i18n/messages_zh.properties b/grails-app/i18n/messages_zh.properties
index f16d2524..bacf856d 100644
--- a/grails-app/i18n/messages_zh.properties
+++ b/grails-app/i18n/messages_zh.properties
@@ -1671,10 +1671,11 @@ provider.group.controller.10=\u8054\u7cfb\u88ab\u521b\u5efa\u4f46\u4e0d\u80fd\u8
 provider.group.controller.11=\u8bf7\u4ece\u73b0\u6709\u7684\u589e\u52a0\u8054\u7cfb.
 provider.group.controller.12=\u6587\u4ef6\u4e0d\u80fd\u4e3a\u7a7a
 provider.group.controller.13=\u4f60\u9009\u7684\u56fe\u50cf\u592a\u5927. \u56fe\u50cf\u88ab\u9650\u5236\u4e3a 200KB.
+gbifCountryToAttribute.label=GBIF Country To Attribute
 
 
 
 
 
 
-#issue types
\ No newline at end of file
+#issue types
diff --git a/grails-app/i18n/messages_zh_CN.properties b/grails-app/i18n/messages_zh_CN.properties
index e81daa29..3e3cba70 100644
--- a/grails-app/i18n/messages_zh_CN.properties
+++ b/grails-app/i18n/messages_zh_CN.properties
@@ -1712,4 +1712,5 @@ datasets.js.facets06=内容类型
 datasets.js.facets07=数据集存在的信息类型. 一个集合可有许多内容类型.
 datasets.js.facets08=含有
 datasets.js.facets09=机构
-datasets.js.facets10=作为数据集的源或管理员的组织或机构
\ No newline at end of file
+datasets.js.facets10=作为数据集的源或管理员的组织或机构
+gbifCountryToAttribute.label=GBIF Country To Attribute
diff --git a/grails-app/init/collectory/BootStrap.groovy b/grails-app/init/collectory/BootStrap.groovy
index 04a113d1..1adc4a08 100644
--- a/grails-app/init/collectory/BootStrap.groovy
+++ b/grails-app/init/collectory/BootStrap.groovy
@@ -1,10 +1,11 @@
 package collectory
 
 class BootStrap {
-     def messageSource
-     def application
+    def messageSource
+    def application
+    def grailsApplication
 
-     def init = { servletContext ->
+    def init = { servletContext ->
         messageSource.setBasenames(
                 "file:///var/opt/atlas/i18n/collectory-plugin/messages",
                 "file:///opt/atlas/i18n/collectory-plugin/messages",
@@ -13,6 +14,10 @@ class BootStrap {
                 "${application.config.biocacheServicesUrl}/facets/i18n"
         )
 
+        // gbifDefaultEntityCountry is mandatory. Not validating this value as 'ZZZ'
+        if (!grailsApplication.config.gbifDefaultEntityCountry) {
+            throw new MissingPropertyException("config `gbifDefaultEntityCountry` is not defined")
+        }
     }
     def destroy = {
     }
diff --git a/grails-app/migrations/5.1.0.2.sql b/grails-app/migrations/5.1.0.2.sql
new file mode 100644
index 00000000..6ff3cb38
--- /dev/null
+++ b/grails-app/migrations/5.1.0.2.sql
@@ -0,0 +1,6 @@
+UPDATE data_provider SET gbif_country_to_attribute = '' WHERE gbif_country_to_attribute IS NULL;
+ALTER TABLE data_provider MODIFY gbif_country_to_attribute varchar(3) NOT NULL;
+
+UPDATE institution SET gbif_country_to_attribute = '' WHERE gbif_country_to_attribute IS NULL;
+ALTER TABLE institution MODIFY gbif_country_to_attribute varchar(3) NOT NULL;
+
diff --git a/grails-app/migrations/changelog.xml b/grails-app/migrations/changelog.xml
index 5ac54ff6..d3ebea2f 100644
--- a/grails-app/migrations/changelog.xml
+++ b/grails-app/migrations/changelog.xml
@@ -441,5 +441,9 @@ liquibase.command.referencePassword: XXXXX
 
     <!-- Table DataLink is no longer required but keep in case of rollback. -->
 
+    <changeSet author="ALA Dev Team" id="20231212-00">
+        <sqlFile path="5.1.0.2.sql" relativeToChangelogFile="true" encoding="ISO-8859-1" stripComments="1" />
+    </changeSet>
+
 
 </databaseChangeLog>
diff --git a/grails-app/services/au/org/ala/collectory/CrudService.groovy b/grails-app/services/au/org/ala/collectory/CrudService.groovy
index aae41b20..0b641bfb 100644
--- a/grails-app/services/au/org/ala/collectory/CrudService.groovy
+++ b/grails-app/services/au/org/ala/collectory/CrudService.groovy
@@ -149,7 +149,7 @@ class CrudService {
     }
 
     def insertDataProvider(obj) {
-        DataProvider dp = new DataProvider(uid: idGeneratorService.getNextDataProviderId())
+        DataProvider dp = new DataProvider(uid: idGeneratorService.getNextDataProviderId(), gbifCountryToAttribute: grailsApplication.config.gbifDefaultEntityCountry)
         updateBaseProperties(dp, obj)
         updateDataProviderProperties(dp, obj)
         dp.userLastModified = obj.user ?: 'Data services'
@@ -598,7 +598,7 @@ class CrudService {
     }
 
     def insertInstitution(obj) {
-        Institution inst = new Institution(uid: idGeneratorService.getNextInstitutionId())
+        Institution inst = new Institution(uid: idGeneratorService.getNextInstitutionId(), gbifCountryToAttribute: grailsApplication.config.gbifDefaultEntityCountry)
         updateBaseProperties(inst, obj)
         updateInstitutionProperties(inst, obj)
         inst.userLastModified = obj.user ?: 'Data services'
diff --git a/grails-app/services/au/org/ala/collectory/DataLoaderService.groovy b/grails-app/services/au/org/ala/collectory/DataLoaderService.groovy
index e88b8d5e..b4ca558d 100644
--- a/grails-app/services/au/org/ala/collectory/DataLoaderService.groovy
+++ b/grails-app/services/au/org/ala/collectory/DataLoaderService.groovy
@@ -11,6 +11,7 @@ class DataLoaderService {
     def institutionCodeLoaderService
     def idGeneratorService
     javax.sql.DataSource dataSource
+    def grailsApplication
 
     boolean transactional = false
 
@@ -55,7 +56,7 @@ class DataLoaderService {
                 DataProvider dp = DataProvider.findByUid(params.uid)
                 if (!dp) {
                     // create it
-                    dp = new DataProvider()
+                    dp = new DataProvider(gbifCountryToAttribute: grailsApplication.config.gbifDefaultEntityCountry)
                     updateProvider dp, params
                     if (!dp.hasErrors() && dp.save(flush: true)) {
                         inserts++
@@ -232,7 +233,7 @@ class DataLoaderService {
                 def originalId = it.id
                 log.info ">>processing ${it.name} original id = ${it.id}"
 
-                Institution ins = new Institution()
+                Institution ins = new Institution(gbifCountryToAttribute: grailsApplication.config.gbifDefaultEntityCountry)
                 ins.name = it.name
                 ins.institutionType = load(it.institutionType)
                 ins.guid = load(it.guid)
@@ -713,7 +714,7 @@ class DataLoaderService {
                 // check whether it's really an institution
                 if (recogniseInstitution(params.name)) {
                     /* provider */
-                    provider = new Institution()
+                    provider = new Institution(gbifCountryToAttribute: grailsApplication.config.gbifDefaultEntityCountry)
                     // load some values
                     provider.properties["guid","name","acronym","focus","notes","websiteUrl","longitude","latitude",
                             "altitude","techDescription","pubDescription"] = params
@@ -817,7 +818,7 @@ class DataLoaderService {
                                 institution.websiteUrl = params.institutionUri
                         } else {
                             log.info ">> Creating institution ${institutionName} for collection ${provider.name}"
-                            institution = new Institution(uid: idGeneratorService.getNextInstitutionId())
+                            institution = new Institution(uid: idGeneratorService.getNextInstitutionId(), gbifCountryToAttribute: grailsApplication.config.gbifDefaultEntityCountry)
                             institution.name = institutionName
                             // fudge the institution guid for now
                             institution.guid = institutionGuid++
diff --git a/grails-app/services/au/org/ala/collectory/GbifRegistryService.groovy b/grails-app/services/au/org/ala/collectory/GbifRegistryService.groovy
index b9042dfe..58ad2fff 100644
--- a/grails-app/services/au/org/ala/collectory/GbifRegistryService.groovy
+++ b/grails-app/services/au/org/ala/collectory/GbifRegistryService.groovy
@@ -740,16 +740,18 @@ class GbifRegistryService {
         }
 
         // convert the 3 digit ISO code to the 2 digit ISO code GBIF needs
-        // Note: GBIF use this for counting "data published by Country X".  There are cases where the postal Address
+        // Note: GBIF use this for counting "data published by Country X". There are cases where the postal Address
         // indicates the headquarters of an international organisation and the country it is located should not be
-        // credited in GBIF as "owning the data".  For those cases, the country is left deliberately null.  This is a
-        // GBIF specific requirement.
+        // credited in GBIF as "owning the data". For International `ZZ` is required.
         organisation.country = null
         if (dp.gbifCountryToAttribute) {
             def iso2 = isoCodeService.iso3CountryCodeToIso2CountryCode(dp.gbifCountryToAttribute.toUpperCase())
             if (iso2) {
                 log.info("Setting GBIF country of attribution to ${iso2}")
                 organisation.country = iso2
+            } else if ("ZZZ".equalsIgnoreCase(dp.gbifCountryToAttribute)) {
+                log.info("Setting GBIF country of attribution to the value for international ${iso2}")
+                organisation.country = 'ZZ'
             }
         }
 
diff --git a/grails-app/views/dataProvider/list.gsp b/grails-app/views/dataProvider/list.gsp
index 5454adce..26be9f3b 100644
--- a/grails-app/views/dataProvider/list.gsp
+++ b/grails-app/views/dataProvider/list.gsp
@@ -39,6 +39,8 @@
                                 <th style="text-align:center;">${message(code: 'dataProvider.gbif.label', default: 'GBIF')}</th>
                             </g:if>
 
+                            <g:sortableColumn property="gbifCountryToAttribute" title="${message(code: 'gbifCountryToAttribute.label', default: 'gbifCountryToAttribute')}" />
+
                         </tr>
                     </thead>
                     <tbody>
@@ -73,6 +75,8 @@
 
                         </g:if>
 
+                          <td>${fieldValue(bean: instance, field: "gbifCountryToAttribute")}</td>
+
                       </tr>
                     </g:each>
                     </tbody>
diff --git a/grails-app/views/shared/_institutionList.gsp b/grails-app/views/shared/_institutionList.gsp
index b000b363..17d09ece 100644
--- a/grails-app/views/shared/_institutionList.gsp
+++ b/grails-app/views/shared/_institutionList.gsp
@@ -22,6 +22,7 @@
                 <g:if test="${grailsApplication.config.gbifRegistrationEnabled == 'true'}">
                     <th style="text-align:center;">${message(code: 'institution.gbif.label', default: 'GBIF')}</th>
                 </g:if>
+                <g:sortableColumn property="gbifCountryToAttribute" title="${message(code: 'gbifCountryToAttribute.label', default: 'gbifCountryToAttribute')}" />
             </tr>
         </thead>
         <tbody>
@@ -65,6 +66,8 @@
 
               </g:if>
 
+              <td>${fieldValue(bean: institutionInstance, field: "gbifCountryToAttribute")}</td>
+
           </tr>
         </g:each>
         </tbody>
diff --git a/grails-app/views/shared/base.gsp b/grails-app/views/shared/base.gsp
index 45725f45..7a951d91 100644
--- a/grails-app/views/shared/base.gsp
+++ b/grails-app/views/shared/base.gsp
@@ -58,11 +58,11 @@
                                   value="${command.resourceType}" />
                     </div>
                 </g:if>
-                <g:if test="${command.ENTITY_TYPE == 'DataProvider'}">
+                <g:if test="${command.ENTITY_TYPE == 'DataProvider' || command.ENTITY_TYPE == 'Institution'}">
                     <div class="form-group">
                         <label for="resourceType"><g:message code="dataprovider.gbif.country" default="GBIF Attribution" /> <cl:helpText code="dataprovider.gbifCountryToAttribute" default="Select the country to attribute within GBIF.org as the publishing country"/></label>
-                        <g:countrySelect id="country"  class="form-control" name="gbifCountryToAttribute" value="${command?.gbifCountryToAttribute}"
-                                         noSelection="['':'-Leave empty for international organisations-']"/>
+                        <g:countrySelect id="country"  class="form-control" name="gbifCountryToAttribute" value="${command?.gbifCountryToAttribute.toLowerCase()}"
+                                         noSelection="['ZZZ':'-Use ZZZ for international organisations-']" required="true"/>
                     </div>
                 </g:if>
                 <g:if test="${command.ENTITY_TYPE == 'Collection'}">
diff --git a/grails-app/views/shared/gbif.gsp b/grails-app/views/shared/gbif.gsp
index 4c1921d6..dcd27d88 100644
--- a/grails-app/views/shared/gbif.gsp
+++ b/grails-app/views/shared/gbif.gsp
@@ -32,10 +32,10 @@
                     <g:textField name="gbifRegistryKey" class="form-control" value="${command?.gbifRegistryKey}" />
                 </div>
 
-                <g:set var="iso" bean="isoCodeService"/>
                 <div class="form-group">
-                    <label for="gbifCountryToAttribute"><g:message code="dataResource.gbif.countryattribute.label" default="GBIF country to attribute data to" /><cl:helpText code="providerGroup.gbifCountryToAttribute"/></label>
-                    <g:select from="${iso.isoCodesMap.entrySet()}" name="gbifCountryToAttribute" value="${command?.gbifCountryToAttribute}"  optionKey="key" optionValue="key"/>
+                    <label for="gbifCountryToAttribute">*<g:message code="dataResource.gbif.countryattribute.label" default="GBIF country to attribute data to" /><cl:helpText code="providerGroup.gbifCountryToAttribute"/></label>
+                    <g:countrySelect id="country"  class="form-control" name="gbifCountryToAttribute" value="${command?.gbifCountryToAttribute?.toLowerCase()}"
+                                     noSelection="['ZZZ':'-Use ZZZ for international organisations-']" required="true"/>
                 </div>
 
                 <div class="buttons">

From af9a699c034bdbdcaa354a6b4c5c6dedf4fe3e78 Mon Sep 17 00:00:00 2001
From: Adam Collins <adam.collins@csiro.au>
Date: Fri, 15 Dec 2023 16:01:26 +1000
Subject: [PATCH 11/13] fix resolveAddress

---
 .../services/au/org/ala/collectory/ProviderGroupService.groovy | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/grails-app/services/au/org/ala/collectory/ProviderGroupService.groovy b/grails-app/services/au/org/ala/collectory/ProviderGroupService.groovy
index 8660d5d2..1b89d90f 100644
--- a/grails-app/services/au/org/ala/collectory/ProviderGroupService.groovy
+++ b/grails-app/services/au/org/ala/collectory/ProviderGroupService.groovy
@@ -147,8 +147,7 @@ class ProviderGroupService {
         def addr = pg.resolveAddress()
         if (!addr && (pg instanceof DataProvider || pg instanceof DataResource)) {
             (pg.consumerInstitutions + pg.consumerCollections).find {
-                def related = _get(it)
-                return related && related.resolveAddress()
+                return it && it.resolveAddress()
             }
         }
         return addr

From 502a801396e6474e95c6ad153158df350e03a8d8 Mon Sep 17 00:00:00 2001
From: Adam Collins <adam.collins@csiro.au>
Date: Thu, 11 Jan 2024 11:00:03 +1000
Subject: [PATCH 12/13] update openapi

---
 build.gradle                                  |  2 +-
 grails-app/conf/application.yml               |  5 +-
 .../org/ala/collectory/DataController.groovy  | 66 +++++++++----------
 .../org/ala/collectory/GbifController.groovy  |  6 +-
 .../org/ala/collectory/IptController.groovy   |  8 +--
 .../ala/collectory/LicenceController.groovy   |  6 +-
 .../ala/collectory/LookupController.groovy    | 18 ++---
 7 files changed, 56 insertions(+), 55 deletions(-)

diff --git a/build.gradle b/build.gradle
index 469bb7bf..3cdf25d0 100644
--- a/build.gradle
+++ b/build.gradle
@@ -114,7 +114,7 @@ dependencies {
     implementation "org.grails.plugins:audit-logging:4.0.3"
     implementation 'dk.glasius:external-config:3.1.1'
     implementation "org.grails.plugins:ala-admin-plugin:2.3.0"
-    implementation 'au.org.ala.plugins:openapi:1.1.0'
+    implementation 'au.org.ala.plugins:openapi:1.3.0'
 
     console "org.grails:grails-console"
     profile "org.grails.profiles:web"
diff --git a/grails-app/conf/application.yml b/grails-app/conf/application.yml
index 95adee0b..6a7e4efc 100644
--- a/grails-app/conf/application.yml
+++ b/grails-app/conf/application.yml
@@ -192,7 +192,7 @@ security:
         enabled: false
     oidc:
         enabled: true
-        discovery-uri: 'https://auth-test.ala.org.au/cas/oidc/.well-known'
+        discoveryUri: 'change me'
         clientId: 'change me'
         secret: 'change me'
         allowUnsignedIdTokens: true
@@ -264,6 +264,7 @@ dataResource:
     contentTypesList:
         - "authoritative"
         - "behaviour"
+        - "citizen science"
         - "commercial uses"
         - "common names"
         - "conservation management"
@@ -448,7 +449,7 @@ environments:
 openapi:
     title: Collectory REST services
     description: REST services for interacting with the collectory webapp
-    terms: https://www.ala.org.au/terms
+    terms: https://www.ala.org.au/terms-of-use
     contact:
         name: Support
         email: support@ala.org.au
diff --git a/grails-app/controllers/au/org/ala/collectory/DataController.groovy b/grails-app/controllers/au/org/ala/collectory/DataController.groovy
index 6074ba25..efb1a87d 100644
--- a/grails-app/controllers/au/org/ala/collectory/DataController.groovy
+++ b/grails-app/controllers/au/org/ala/collectory/DataController.groovy
@@ -270,9 +270,9 @@ class DataController {
                                     )
                             ],
                             headers = [
-                                    @Header(name = 'Access-Control-Allow-Headers', description = "CORS header", schema = @Schema(type = "String")),
-                                    @Header(name = 'Access-Control-Allow-Methods', description = "CORS header", schema = @Schema(type = "String")),
-                                    @Header(name = 'Access-Control-Allow-Origin', description = "CORS header", schema = @Schema(type = "String"))
+                                    @Header(name = 'Access-Control-Allow-Headers', description = "CORS header", schema = @Schema(type = "string")),
+                                    @Header(name = 'Access-Control-Allow-Methods', description = "CORS header", schema = @Schema(type = "string")),
+                                    @Header(name = 'Access-Control-Allow-Origin', description = "CORS header", schema = @Schema(type = "string"))
                             ]
                     )
             ],
@@ -352,9 +352,9 @@ class DataController {
                                     )
                             ],
                             headers = [
-                                    @Header(name = 'Access-Control-Allow-Headers', description = "CORS header", schema = @Schema(type = "String")),
-                                    @Header(name = 'Access-Control-Allow-Methods', description = "CORS header", schema = @Schema(type = "String")),
-                                    @Header(name = 'Access-Control-Allow-Origin', description = "CORS header", schema = @Schema(type = "String"))
+                                    @Header(name = 'Access-Control-Allow-Headers', description = "CORS header", schema = @Schema(type = "string")),
+                                    @Header(name = 'Access-Control-Allow-Methods', description = "CORS header", schema = @Schema(type = "string")),
+                                    @Header(name = 'Access-Control-Allow-Origin', description = "CORS header", schema = @Schema(type = "string"))
                             ]
                     )
             ],
@@ -477,9 +477,9 @@ class DataController {
                                     )
                             ],
                             headers = [
-                                    @Header(name = 'Access-Control-Allow-Headers', description = "CORS header", schema = @Schema(type = "String")),
-                                    @Header(name = 'Access-Control-Allow-Methods', description = "CORS header", schema = @Schema(type = "String")),
-                                    @Header(name = 'Access-Control-Allow-Origin', description = "CORS header", schema = @Schema(type = "String"))
+                                    @Header(name = 'Access-Control-Allow-Headers', description = "CORS header", schema = @Schema(type = "string")),
+                                    @Header(name = 'Access-Control-Allow-Methods', description = "CORS header", schema = @Schema(type = "string")),
+                                    @Header(name = 'Access-Control-Allow-Origin', description = "CORS header", schema = @Schema(type = "string"))
                             ]
                     )
             ],
@@ -577,9 +577,9 @@ class DataController {
                                     )
                             ],
                             headers = [
-                                    @Header(name = 'Access-Control-Allow-Headers', description = "CORS header", schema = @Schema(type = "String")),
-                                    @Header(name = 'Access-Control-Allow-Methods', description = "CORS header", schema = @Schema(type = "String")),
-                                    @Header(name = 'Access-Control-Allow-Origin', description = "CORS header", schema = @Schema(type = "String"))
+                                    @Header(name = 'Access-Control-Allow-Headers', description = "CORS header", schema = @Schema(type = "string")),
+                                    @Header(name = 'Access-Control-Allow-Methods', description = "CORS header", schema = @Schema(type = "string")),
+                                    @Header(name = 'Access-Control-Allow-Origin', description = "CORS header", schema = @Schema(type = "string"))
                             ]
                     )
             ],
@@ -653,9 +653,9 @@ class DataController {
                                     )
                             ],
                             headers = [
-                                    @Header(name = 'Access-Control-Allow-Headers', description = "CORS header", schema = @Schema(type = "String")),
-                                    @Header(name = 'Access-Control-Allow-Methods', description = "CORS header", schema = @Schema(type = "String")),
-                                    @Header(name = 'Access-Control-Allow-Origin', description = "CORS header", schema = @Schema(type = "String"))
+                                    @Header(name = 'Access-Control-Allow-Headers', description = "CORS header", schema = @Schema(type = "string")),
+                                    @Header(name = 'Access-Control-Allow-Methods', description = "CORS header", schema = @Schema(type = "string")),
+                                    @Header(name = 'Access-Control-Allow-Origin', description = "CORS header", schema = @Schema(type = "string"))
                             ]
                     )
             ],
@@ -734,9 +734,9 @@ class DataController {
                                     )
                             ],
                             headers = [
-                                    @Header(name = 'Access-Control-Allow-Headers', description = "CORS header", schema = @Schema(type = "String")),
-                                    @Header(name = 'Access-Control-Allow-Methods', description = "CORS header", schema = @Schema(type = "String")),
-                                    @Header(name = 'Access-Control-Allow-Origin', description = "CORS header", schema = @Schema(type = "String"))
+                                    @Header(name = 'Access-Control-Allow-Headers', description = "CORS header", schema = @Schema(type = "string")),
+                                    @Header(name = 'Access-Control-Allow-Methods', description = "CORS header", schema = @Schema(type = "string")),
+                                    @Header(name = 'Access-Control-Allow-Origin', description = "CORS header", schema = @Schema(type = "string"))
                             ]
                     )
             ],
@@ -927,9 +927,9 @@ class DataController {
                                     )
                             ],
                             headers = [
-                                    @Header(name = 'Access-Control-Allow-Headers', description = "CORS header", schema = @Schema(type = "String")),
-                                    @Header(name = 'Access-Control-Allow-Methods', description = "CORS header", schema = @Schema(type = "String")),
-                                    @Header(name = 'Access-Control-Allow-Origin', description = "CORS header", schema = @Schema(type = "String"))
+                                    @Header(name = 'Access-Control-Allow-Headers', description = "CORS header", schema = @Schema(type = "string")),
+                                    @Header(name = 'Access-Control-Allow-Methods', description = "CORS header", schema = @Schema(type = "string")),
+                                    @Header(name = 'Access-Control-Allow-Origin', description = "CORS header", schema = @Schema(type = "string"))
                             ]
                     )
             ],
@@ -1082,9 +1082,9 @@ class DataController {
                                     )
                             ],
                             headers = [
-                                    @Header(name = 'Access-Control-Allow-Headers', description = "CORS header", schema = @Schema(type = "String")),
-                                    @Header(name = 'Access-Control-Allow-Methods', description = "CORS header", schema = @Schema(type = "String")),
-                                    @Header(name = 'Access-Control-Allow-Origin', description = "CORS header", schema = @Schema(type = "String"))
+                                    @Header(name = 'Access-Control-Allow-Headers', description = "CORS header", schema = @Schema(type = "string")),
+                                    @Header(name = 'Access-Control-Allow-Methods', description = "CORS header", schema = @Schema(type = "string")),
+                                    @Header(name = 'Access-Control-Allow-Origin', description = "CORS header", schema = @Schema(type = "string"))
                             ]
                     )
             ],
@@ -1193,9 +1193,9 @@ class DataController {
                                     )
                             ],
                             headers = [
-                                    @Header(name = 'Access-Control-Allow-Headers', description = "CORS header", schema = @Schema(type = "String")),
-                                    @Header(name = 'Access-Control-Allow-Methods', description = "CORS header", schema = @Schema(type = "String")),
-                                    @Header(name = 'Access-Control-Allow-Origin', description = "CORS header", schema = @Schema(type = "String"))
+                                    @Header(name = 'Access-Control-Allow-Headers', description = "CORS header", schema = @Schema(type = "string")),
+                                    @Header(name = 'Access-Control-Allow-Methods', description = "CORS header", schema = @Schema(type = "string")),
+                                    @Header(name = 'Access-Control-Allow-Origin', description = "CORS header", schema = @Schema(type = "string"))
                             ]
                     )
             ],
@@ -1511,9 +1511,9 @@ class DataController {
                                     )
                             ],
                             headers = [
-                                    @Header(name = 'Access-Control-Allow-Headers', description = "CORS header", schema = @Schema(type = "String")),
-                                    @Header(name = 'Access-Control-Allow-Methods', description = "CORS header", schema = @Schema(type = "String")),
-                                    @Header(name = 'Access-Control-Allow-Origin', description = "CORS header", schema = @Schema(type = "String"))
+                                    @Header(name = 'Access-Control-Allow-Headers', description = "CORS header", schema = @Schema(type = "string")),
+                                    @Header(name = 'Access-Control-Allow-Methods', description = "CORS header", schema = @Schema(type = "string")),
+                                    @Header(name = 'Access-Control-Allow-Origin', description = "CORS header", schema = @Schema(type = "string"))
                             ]
                     ),
                     @ApiResponse(
@@ -1526,9 +1526,9 @@ class DataController {
                                     )
                             ],
                             headers = [
-                                    @Header(name = 'Access-Control-Allow-Headers', description = "CORS header", schema = @Schema(type = "String")),
-                                    @Header(name = 'Access-Control-Allow-Methods', description = "CORS header", schema = @Schema(type = "String")),
-                                    @Header(name = 'Access-Control-Allow-Origin', description = "CORS header", schema = @Schema(type = "String"))
+                                    @Header(name = 'Access-Control-Allow-Headers', description = "CORS header", schema = @Schema(type = "string")),
+                                    @Header(name = 'Access-Control-Allow-Methods', description = "CORS header", schema = @Schema(type = "string")),
+                                    @Header(name = 'Access-Control-Allow-Origin', description = "CORS header", schema = @Schema(type = "string"))
                             ]
                     )
             ],
diff --git a/grails-app/controllers/au/org/ala/collectory/GbifController.groovy b/grails-app/controllers/au/org/ala/collectory/GbifController.groovy
index 2d91f06d..fcb87569 100644
--- a/grails-app/controllers/au/org/ala/collectory/GbifController.groovy
+++ b/grails-app/controllers/au/org/ala/collectory/GbifController.groovy
@@ -181,9 +181,9 @@ class GbifController {
                                     )
                             ],
                             headers = [
-                                    @Header(name = 'Access-Control-Allow-Headers', description = "CORS header", schema = @Schema(type = "String")),
-                                    @Header(name = 'Access-Control-Allow-Methods', description = "CORS header", schema = @Schema(type = "String")),
-                                    @Header(name = 'Access-Control-Allow-Origin', description = "CORS header", schema = @Schema(type = "String"))
+                                    @Header(name = 'Access-Control-Allow-Headers', description = "CORS header", schema = @Schema(type = "string")),
+                                    @Header(name = 'Access-Control-Allow-Methods', description = "CORS header", schema = @Schema(type = "string")),
+                                    @Header(name = 'Access-Control-Allow-Origin', description = "CORS header", schema = @Schema(type = "string"))
                             ]
                     )
             ],
diff --git a/grails-app/controllers/au/org/ala/collectory/IptController.groovy b/grails-app/controllers/au/org/ala/collectory/IptController.groovy
index 34c670a9..e69ff503 100644
--- a/grails-app/controllers/au/org/ala/collectory/IptController.groovy
+++ b/grails-app/controllers/au/org/ala/collectory/IptController.groovy
@@ -118,9 +118,9 @@ class IptController {
                                     )
                             ],
                             headers = [
-                                    @Header(name = 'Access-Control-Allow-Headers', description = "CORS header", schema = @Schema(type = "String")),
-                                    @Header(name = 'Access-Control-Allow-Methods', description = "CORS header", schema = @Schema(type = "String")),
-                                    @Header(name = 'Access-Control-Allow-Origin', description = "CORS header", schema = @Schema(type = "String"))
+                                    @Header(name = 'Access-Control-Allow-Headers', description = "CORS header", schema = @Schema(type = "string")),
+                                    @Header(name = 'Access-Control-Allow-Methods', description = "CORS header", schema = @Schema(type = "string")),
+                                    @Header(name = 'Access-Control-Allow-Origin', description = "CORS header", schema = @Schema(type = "string"))
                             ]
                     )
             ],
@@ -226,4 +226,4 @@ class IptController {
             csvWriter.flush()
         }
     }
-}
\ No newline at end of file
+}
diff --git a/grails-app/controllers/au/org/ala/collectory/LicenceController.groovy b/grails-app/controllers/au/org/ala/collectory/LicenceController.groovy
index 7519fc09..5527553b 100644
--- a/grails-app/controllers/au/org/ala/collectory/LicenceController.groovy
+++ b/grails-app/controllers/au/org/ala/collectory/LicenceController.groovy
@@ -37,9 +37,9 @@ class LicenceController {
                                     )
                             ],
                             headers = [
-                                    @Header(name = 'Access-Control-Allow-Headers', description = "CORS header", schema = @Schema(type = "String")),
-                                    @Header(name = 'Access-Control-Allow-Methods', description = "CORS header", schema = @Schema(type = "String")),
-                                    @Header(name = 'Access-Control-Allow-Origin', description = "CORS header", schema = @Schema(type = "String"))
+                                    @Header(name = 'Access-Control-Allow-Headers', description = "CORS header", schema = @Schema(type = "string")),
+                                    @Header(name = 'Access-Control-Allow-Methods', description = "CORS header", schema = @Schema(type = "string")),
+                                    @Header(name = 'Access-Control-Allow-Origin', description = "CORS header", schema = @Schema(type = "string"))
                             ]
                     )
             ],
diff --git a/grails-app/controllers/au/org/ala/collectory/LookupController.groovy b/grails-app/controllers/au/org/ala/collectory/LookupController.groovy
index bce08e5e..6c1c02a1 100644
--- a/grails-app/controllers/au/org/ala/collectory/LookupController.groovy
+++ b/grails-app/controllers/au/org/ala/collectory/LookupController.groovy
@@ -71,9 +71,9 @@ class LookupController {
                                     )
                             ],
                             headers = [
-                                    @Header(name = 'Access-Control-Allow-Headers', description = "CORS header", schema = @Schema(type = "String")),
-                                    @Header(name = 'Access-Control-Allow-Methods', description = "CORS header", schema = @Schema(type = "String")),
-                                    @Header(name = 'Access-Control-Allow-Origin', description = "CORS header", schema = @Schema(type = "String"))
+                                    @Header(name = 'Access-Control-Allow-Headers', description = "CORS header", schema = @Schema(type = "string")),
+                                    @Header(name = 'Access-Control-Allow-Methods', description = "CORS header", schema = @Schema(type = "string")),
+                                    @Header(name = 'Access-Control-Allow-Origin', description = "CORS header", schema = @Schema(type = "string"))
                             ]
                     )
             ],
@@ -210,9 +210,9 @@ class LookupController {
                                     )
                             ],
                             headers = [
-                                    @Header(name = 'Access-Control-Allow-Headers', description = "CORS header", schema = @Schema(type = "String")),
-                                    @Header(name = 'Access-Control-Allow-Methods', description = "CORS header", schema = @Schema(type = "String")),
-                                    @Header(name = 'Access-Control-Allow-Origin', description = "CORS header", schema = @Schema(type = "String"))
+                                    @Header(name = 'Access-Control-Allow-Headers', description = "CORS header", schema = @Schema(type = "string")),
+                                    @Header(name = 'Access-Control-Allow-Methods', description = "CORS header", schema = @Schema(type = "string")),
+                                    @Header(name = 'Access-Control-Allow-Origin', description = "CORS header", schema = @Schema(type = "string"))
                             ]
                     )
             ],
@@ -393,9 +393,9 @@ class LookupController {
                                     )
                             ],
                             headers = [
-                                    @Header(name = 'Access-Control-Allow-Headers', description = "CORS header", schema = @Schema(type = "String")),
-                                    @Header(name = 'Access-Control-Allow-Methods', description = "CORS header", schema = @Schema(type = "String")),
-                                    @Header(name = 'Access-Control-Allow-Origin', description = "CORS header", schema = @Schema(type = "String"))
+                                    @Header(name = 'Access-Control-Allow-Headers', description = "CORS header", schema = @Schema(type = "string")),
+                                    @Header(name = 'Access-Control-Allow-Methods', description = "CORS header", schema = @Schema(type = "string")),
+                                    @Header(name = 'Access-Control-Allow-Origin', description = "CORS header", schema = @Schema(type = "string"))
                             ]
                     )
             ],

From 9424bda9dc1c24457ccfa8ff3dbb7dee4c6c7b18 Mon Sep 17 00:00:00 2001
From: Adam Collins <adam.collins@csiro.au>
Date: Mon, 15 Jan 2024 14:33:52 +1000
Subject: [PATCH 13/13] datahub fix

---
 grails-app/domain/au/org/ala/collectory/DataHub.groovy |  4 +++-
 grails-app/migrations/5.1.0.3.sql                      | 10 ++++++++++
 grails-app/migrations/changelog.xml                    |  4 ++++
 .../au/org/ala/collectory/CollectoryTagLib.groovy      |  2 +-
 4 files changed, 18 insertions(+), 2 deletions(-)
 create mode 100644 grails-app/migrations/5.1.0.3.sql

diff --git a/grails-app/domain/au/org/ala/collectory/DataHub.groovy b/grails-app/domain/au/org/ala/collectory/DataHub.groovy
index 6bcb6683..21068d2a 100644
--- a/grails-app/domain/au/org/ala/collectory/DataHub.groovy
+++ b/grails-app/domain/au/org/ala/collectory/DataHub.groovy
@@ -15,6 +15,8 @@ class DataHub implements ProviderGroup, Serializable {
     String members = '[]'                  // non-overlapping json list of uids of member institutions and collections
                                            //  (suitable for identifying a unique list of occurrence records)
 
+    static hasMany = [externalIdentifiers: ExternalIdentifier]
+
     static constraints = {
         guid(nullable:true, maxSize:256)
         uid(blank:false, maxSize:20)
@@ -63,7 +65,7 @@ class DataHub implements ProviderGroup, Serializable {
     }
 
     static transients =  ['collectionMember', 'institutionMember', 'dataResourceMember']
-    
+
     boolean canBeMapped() {
         return false;
     }
diff --git a/grails-app/migrations/5.1.0.3.sql b/grails-app/migrations/5.1.0.3.sql
new file mode 100644
index 00000000..f4e74e5d
--- /dev/null
+++ b/grails-app/migrations/5.1.0.3.sql
@@ -0,0 +1,10 @@
+CREATE TABLE `data_hub_external_identifier` (
+                                                `data_hub_external_identifiers_id` bigint NOT NULL,
+                                                `external_identifier_id` bigint DEFAULT NULL,
+                                                KEY `FKirj1ih9m2xspn1aki6xoeavsx` (`external_identifier_id`),
+    KEY `FKiwkh0on1fsiy5on971yoi0e0q` (`data_hub_external_identifiers_id`),
+    CONSTRAINT `FKirj1ih9m2xspn1aki6xoeavsx` FOREIGN KEY (`external_identifier_id`) REFERENCES `external_identifier` (`id`),
+    CONSTRAINT `FKiwkh0on1fsiy5on971yoi0e0q` FOREIGN KEY (`data_hub_external_identifiers_id`) REFERENCES `data_hub` (`id`)
+    ) ENGINE=InnoDB DEFAULT CHARSET=utf8;
+
+insert into `data_hub_external_identifier` (`data_hub_external_identifiers_id`, `external_identifier_id`) select c.id, e.id from external_identifier e inner join data_hub c on e.entity_uid = c.uid;
diff --git a/grails-app/migrations/changelog.xml b/grails-app/migrations/changelog.xml
index d3ebea2f..95176f4e 100644
--- a/grails-app/migrations/changelog.xml
+++ b/grails-app/migrations/changelog.xml
@@ -445,5 +445,9 @@ liquibase.command.referencePassword: XXXXX
         <sqlFile path="5.1.0.2.sql" relativeToChangelogFile="true" encoding="ISO-8859-1" stripComments="1" />
     </changeSet>
 
+    <changeSet author="ALA Dev Team" id="20240115-00">
+        <sqlFile path="5.1.0.3.sql" relativeToChangelogFile="true" encoding="ISO-8859-1" stripComments="1" />
+    </changeSet>
+
 
 </databaseChangeLog>
diff --git a/grails-app/taglib/au/org/ala/collectory/CollectoryTagLib.groovy b/grails-app/taglib/au/org/ala/collectory/CollectoryTagLib.groovy
index 7dee5244..29d39c39 100644
--- a/grails-app/taglib/au/org/ala/collectory/CollectoryTagLib.groovy
+++ b/grails-app/taglib/au/org/ala/collectory/CollectoryTagLib.groovy
@@ -1468,7 +1468,7 @@ class CollectoryTagLib {
             out << "</ul></p>\n"
         }
         def consumers = []
-        if (attrs.instance instanceof DataProvider || attrs.instance instanceof DataResource) {
+        if (attrs.instance instanceof DataResource) {
             consumers += attrs.instance.consumerInstitutions + attrs.instance.consumerCollections
             if (attrs.instance.dataProvider) {
                 consumers += attrs.instance.dataProvider.consumerInstitutions + attrs.instance.dataProvider.consumerCollections