Merge pull request #6 from biodiversitydata-se/repatriate2

Improve repatriate functionality

grails-app/controllers/au/org/ala/collectory/ManageController.groovy

@@ -38,12 +38,15 @@ class ManageController {
                 endpoint: new URL(grailsApplication.config.gbifApiUrl),
                 username: '',
                 password: '',
-                country: Locale.default.getCountry(),
+                country: grailsApplication.config.getProperty(
+                        'repatriate.defaults.country', String.class, Locale.default.getCountry()),
                 recordType: 'OCCURRENCE',
                 defaultDatasetValues: [:],
                 keyTerms: [],
                 resources: [],
-                countries: gbifService.getCountryMap().keySet()
+                countries: gbifService.getCountryMap().keySet(),
+                dataProviderUid: grailsApplication.config.getProperty(
+                        'repatriate.defaults.dataProviderUId', String.class)
         )
         def adaptor = configuration.createAdaptor()
         render(view: "repatriate",

grails-app/domain/au/org/ala/collectory/ExternalIdentifier.groovy

@@ -22,6 +22,10 @@ class ExternalIdentifier {
         return "${source}:${identifier}"
     }
 
+    boolean isValidUri() {
+        uri?.startsWith("http://") || uri?.startsWith("https://")
+    }
+
     /**
      * Is this the same identifier?
      *

grails-app/services/au/org/ala/collectory/CrudService.groovy

@@ -402,6 +402,7 @@ class CrudService {
             DataResource dr = new DataResource(uid: idGeneratorService.getNextDataResourceId())
             updateBaseProperties(dr, obj)
             updateDataResourceProperties(dr, obj)
+            dr.externalIdentifiers = []
             dr.userLastModified = obj.user ?: 'Data services'
             if (!dr.hasErrors()) {
                 DataResource.withTransaction {

grails-app/services/au/org/ala/collectory/ExternalDataService.groovy

@@ -162,7 +162,7 @@ class ExternalDataService {
                 if (dr.hasErrors()) {
                     throw new ExternalResourceException("Created resource has errors", "manage.note.note02", resource.name, dr.errors)
                 }
-                dr.addExternalIdentifier(resource.guid, adaptor.source, resource.source)
+                externalIdentifierService.addExternalIdentifier(dr.uid, resource.guid, adaptor.source, resource.source)
                 resource.uid = dr.uid
                 resource.addNote("manage.note.note03", resource.uid)
             }

grails-app/views/manage/repatriate.gsp

@@ -30,11 +30,11 @@
             </div>
             <div class="form-group">
                 <label for="country"><g:message code="manage.repatriationCountry.label06" /><cl:helpText code="manage.extload.label06.help"/></label>
-                <g:select name="country" class="form-control" from="${countryMap.entrySet()}" optionKey="key" optionValue="value" values="${configuration.country}" value="${params.country}"/>
+                <g:select name="country" class="form-control" from="${countryMap.entrySet()}" optionKey="key" optionValue="value" value="${configuration.country}"/>
             </div>
             <div class="form-group hide">
                 <label for="recordType"><g:message code="manage.extload.label07" /><cl:helpText code="manage.extload.label07.help"/></label>
-                <g:select name="recordType" class="form-control" from="${datasetTypeMap.entrySet()}" optionKey="key" optionValue="value" values="${configuration.recordType}"/>
+                <g:select name="recordType" class="form-control" from="${datasetTypeMap.entrySet()}" optionKey="key" optionValue="value" value="${configuration.recordType}"/>
             </div>
             <div class="form-group hide">
                 <label for="name"><g:message code="manage.extload.label01" /><cl:helpText code="manage.extload.label01.help"/></label>

grails-app/views/manage/repatriateReview.gsp

@@ -6,7 +6,7 @@
           content="${createLink(action: 'list', controller: 'manage')},${message(code: 'manage.list.title01')}"
     />
     <meta name="breadcrumbs"
-          content="${createLink(action: 'list', controller: 'manage')},Repatriation tools"
+          content="${createLink(action: 'repatriate', controller: 'manage')},Repatriation tools"
     />
     <meta name="layout" content="${grailsApplication.config.skin.layout}" />
     <title><g:message code="manage.extloadr.title" /></title>
@@ -80,7 +80,7 @@
                 <td><g:checkBox name="resources[${rs}].addResource" value="${res.addResource}"/></td>
                 <td><g:checkBox name="resources[${rs}].updateMetadata" value="${res.updateMetadata}"/></td>
                 <td><g:checkBox name="resources[${rs}].updateConnection" value="${res.updateConnection}"/></td>
-                <td>${res.recordCount}</td>
+                <td><g:formatNumber number="${res.recordCount}" format="###,###,##0" /></td>
             </tr>
             </g:each>
             </g:if>
@@ -139,7 +139,7 @@
                 {"orderable": false},
                 {"orderable": false},
                 {"orderable": false},
-                {"orderable": false}
+                null
             ]
         });
     } );

grails-app/views/public/_externalIdentifiers.gsp

@@ -4,7 +4,7 @@
         <h4><g:message code="public.sdr.externalIdentifiers.title" /></h4>
         <ul>
             <g:each var="ext" in="${external}">
-                <li><g:if test="${ext.uri}"><a href="${ext.uri}" class="external" target="_blank"><g:fieldValue bean="${ext}" field="label"/></a></g:if><g:else><g:fieldValue bean="${ext}" field="label"/></g:else></li>
+                <li><g:if test="${ext.validUri}"><a href="${ext.uri}" class="external" target="_blank"><g:fieldValue bean="${ext}" field="label"/></a></g:if><g:else><g:fieldValue bean="${ext}" field="label"/></g:else></li>
             </g:each>
         </ul>
     </section>

sbdi/data/config/collectory-config.properties

@@ -122,7 +122,7 @@ skin.taxaLinks.baseUrl = https://species.biodiversitydata.se/species/
 spatial.baseURL = https://spatial.biodiversitydata.se/
 
 # GBIF base URL for webservices
-gbifApiUrl=http://api.gbif.org/v1
+gbifApiUrl=https://api.gbif.org/v1/
 
 # GBIF Registration enables the Collectory to create and update organisations and datasets
 # This mode of operation is only expected to be used by National Nodes running the ALA as a publishing gateway to GBIF.
@@ -202,3 +202,6 @@ userdetails.api.url=https://auth.biodiversitydata.se/userdetails/
 gbifDefaultEntityCountry=SWE
 
 sitemap.dir=/data/collectory/data/sitemap
+
+repatriate.defaults.country=SE
+repatriate.defaults.dataProviderUId=dp4

src/main/groovy/au/org/ala/collectory/resources/gbif/GbifDataSourceAdapter.groovy

@@ -33,7 +33,7 @@ class GbifDataSourceAdapter extends DataSourceAdapter {
     static final MessageFormat DATASET_GET = new MessageFormat("dataset/{0}")
     static final MessageFormat DATASET_RECORD_COUNT = new MessageFormat("occurrence/count?datasetKey={0}")
     static final MessageFormat DOWNLOAD_STATUS = new MessageFormat("occurrence/download/{0}")
-    static final DateFormat TIMESTAMP_FORMAT= new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSZ")
+    static final DateFormat TIMESTAMP_FORMAT= new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSXXX")
 
     static LICENSE_MAP = [
             "https://creativecommons.org/publicdomain/zero/1.0/legalcode": [licenseType: "CC0", licenseVersion: "1.0" ],
@@ -44,19 +44,19 @@ class GbifDataSourceAdapter extends DataSourceAdapter {
             "http://creativecommons.org/licenses/by/4.0/legalcode":        [licenseType: "CC BY", licenseVersion: "4.0" ]
     ]
     static TYPE_MAP = [
-            "CHECKLIST"    : "species-list",
-            "METADATA"     : "document",
-            "OCCURRENCE"   : "records",
-            "SAPLING_EVENT": "records"
+            "CHECKLIST"     : "species-list",
+            "METADATA"      : "document",
+            "OCCURRENCE"    : "records",
+            "SAMPLING_EVENT": "records"
     ]
     static DATASET_TYPES = [
             "OCCURRENCE"   : "Occurrence Records"  // We only allow occurrence records at the moment
     ]
     static CONTENT_MAP = [
-            "CHECKLIST"    : ["species list", "taxonomy", "gbif import"],
-            "METADATA"     : ["gbif import"],
-            "OCCURRENCE"   : ["point occurrence data", "gbif import"],
-            "SAPLING_EVENT": ["point occurrence data", "gbif import"]
+            "CHECKLIST"     : ["species list", "taxonomy", "gbif import"],
+            "METADATA"      : ["gbif import"],
+            "OCCURRENCE"    : ["point occurrence data", "gbif import"],
+            "SAMPLING_EVENT": ["point occurrence data", "gbif import"]
     ]
     static DOWNLOAD_STATUS_MAP = [
             "CANCELLED" : TaskPhase.CANCELLED,
@@ -195,7 +195,8 @@ class GbifDataSourceAdapter extends DataSourceAdapter {
         def contentTypes = CONTENT_MAP[dataset.type] ?: []
         def source = dataset.doi?.replace("doi:", "https://doi.org/")
         try {
-            currency = dataset.pubDate ? TIMESTAMP_FORMAT.clone().parse(dataset.pubDate) : null
+            def unparsedCurrency = dataset.pubDate ?: dataset.modified
+            currency = unparsedCurrency ? TIMESTAMP_FORMAT.clone().parse(unparsedCurrency) : null
         } catch (ParseException ex) {
         }
         def resource = [
@@ -205,7 +206,10 @@ class GbifDataSourceAdapter extends DataSourceAdapter {
                 address: address,
                 phone: phone,
                 email: email,
-                pubDescription: dataset.description,
+                pubDescription: dataset.description
+                        .replaceAll("(?i)<p[^>]*>", "\n")
+                        .replaceAll("(?i)</p>", "")
+                        .replaceAll("(?i)<br\\s*/?>", "\n"),
                 state: address?.state,
                 websiteUrl: dataset.homepage,
                 rights: dataset.rights,

src/main/groovy/au/org/ala/collectory/resources/gbif/GbifRepatDataSourceAdapter.groovy

@@ -17,16 +17,24 @@ class GbifRepatDataSourceAdapter extends GbifDataSourceAdapter {
     static final SOURCE = "GBIF_REPATRIATION"
     GbifService gbifService
 
-    static final String OCCURRENCE_REPAT_SEARCH = "occurrence/search?repatriated=true&country={0}&type={1}&offset=0&limit=0&facet=datasetKey&facetLimit=10000"
+    static final String OCCURRENCE_REPAT_SEARCH = "occurrence/search?country={0}&type={1}&offset=0&limit=0&facet=datasetKey&facetLimit=10000"
+    static final String COUNTRY_DATASETS = "dataset?country={0}&limit=10000"
 
     GbifRepatDataSourceAdapter(DataSourceConfiguration configuration) {
         super(configuration)
      }
 
+    List<String> fetchCountryDatasetKeys() {
+        String url = MessageFormat.format(COUNTRY_DATASETS, configuration.country)
+        JSONObject json = getJSONWS(url, false)
+        json.results.collect{ it -> it.key }
+    }
+
     @Override
     List<Map> datasets() throws ExternalResourceException {
         def keys = []
         def datasets = []
+        def countryDatasetKeys = fetchCountryDatasetKeys()
 
         LOGGER.info("Requesting dataset lists configuration.country: ${configuration.country}")
         String url = MessageFormat.format(OCCURRENCE_REPAT_SEARCH, configuration.country, configuration.recordType)
@@ -35,7 +43,10 @@ class GbifRepatDataSourceAdapter extends GbifDataSourceAdapter {
             json.facets[0].counts.each {
                 keys << it.name
 
-                if (it.count >= configuration.minRecordCount && it.count <= configuration.maxRecordCount && datasets.size() < configuration.maxNoOfDatasets) {
+                if (!countryDatasetKeys.contains(it.name) &&
+                        it.count >= configuration.minRecordCount &&
+                        it.count <= configuration.maxRecordCount &&
+                        datasets.size() < configuration.maxNoOfDatasets) {
                     LOGGER.info("Getting metadata for ${it.name}  = ${it.count}")
                     def dataset = getDataset(it.name, it.count)
                     if (dataset.name) {