Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Improve repatriate functionality (old) #5

Closed
wants to merge 9 commits into from
Closed

Improve repatriate functionality (old) #5

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
e46d7a0
Fix issue with creating an external data resource
matsbov Jul 11, 2024
5aaf0c4
Add settings for default values on repatriate search view
matsbov Jul 11, 2024
c51d983
Add type to ExternalResourceBean
matsbov Jul 11, 2024
3cec4d6
Filter repatriated datasets against country datasets (do not use repa…
matsbov Jul 11, 2024
7264641
Fix issues and improve the repatriate review page
matsbov Jul 11, 2024
36caf96
Do not link invalid urls on data resource page
matsbov Jul 12, 2024
96419d2
Fix timestamp parsing
matsbov Jul 12, 2024
d7dc42b
Strip p and br tags in description
matsbov Jul 12, 2024
d83fb61
Update gbif api url
matsbov Jul 12, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 12 additions & 3 deletions grails-app/controllers/au/org/ala/collectory/ManageController.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,12 +38,15 @@ class ManageController {
endpoint: new URL(grailsApplication.config.gbifApiUrl),
username: '',
password: '',
country: Locale.default.getCountry(),
country: grailsApplication.config.getProperty(
'repatriate.defaults.country', String.class, Locale.default.getCountry()),
recordType: 'OCCURRENCE',
defaultDatasetValues: [:],
keyTerms: [],
resources: [],
countries: gbifService.getCountryMap().keySet()
countries: gbifService.getCountryMap().keySet(),
dataProviderUid: grailsApplication.config.getProperty(
'repatriate.defaults.dataProviderUId', String.class)
)
def adaptor = configuration.createAdaptor()
render(view: "repatriate",
Expand All @@ -53,7 +56,13 @@ class ManageController {
countryMap: gbifService.getCountryMap(),
datasetTypeMap: adaptor.datasetTypeMap,
adaptors: externalDataService.REPAT_ADAPTORMAP,
dataProviders: DataProvider.all.sort { it.name }
dataProviders: DataProvider.all.sort { it.name },
maxDatasetCount: grailsApplication.config.getProperty(
'repatriate.defaults.maxDatasetCount', Integer.class, 25),
minRecordCount: grailsApplication.config.getProperty(
'repatriate.defaults.minRecordCount', Integer.class, 10000),
maxRecordCount: grailsApplication.config.getProperty(
'repatriate.defaults.maxRecordCount', Integer.class, 1000000),
]
)
}
Expand Down
4 changes: 4 additions & 0 deletions grails-app/domain/au/org/ala/collectory/ExternalIdentifier.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,10 @@ class ExternalIdentifier {
return "${source}:${identifier}"
}

boolean isValidUri() {
uri?.startsWith("http://") || uri?.startsWith("https://")
}

/**
* Is this the same identifier?
*
Expand Down
2 changes: 1 addition & 1 deletion grails-app/domain/au/org/ala/collectory/ExternalResourceBean.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,8 +44,8 @@ class ExternalResourceBean implements Comparable<ExternalResourceBean> {
String occurrenceId
/** The number of records in this resource */
Integer recordCount
/** The number of records in this resource */
String country
String type

/**
* Add a note to the resource bean load
Expand Down
1 change: 1 addition & 0 deletions grails-app/i18n/messages.properties
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1369,6 +1369,7 @@ manage.extloadr.label06=Add Resource
manage.extloadr.label07=Update Metadata
manage.extloadr.label08=(Re) download dataset
manage.extloadr.label09=Record count
manage.extloadr.label10=Type
manage.extloadr.button01=...
manage.extloadr.button.cancel=Cancel
manage.extloadr.button.ok=Ok
Expand Down
1 change: 1 addition & 0 deletions grails-app/services/au/org/ala/collectory/CrudService.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -402,6 +402,7 @@ class CrudService {
DataResource dr = new DataResource(uid: idGeneratorService.getNextDataResourceId())
updateBaseProperties(dr, obj)
updateDataResourceProperties(dr, obj)
dr.externalIdentifiers = []
dr.userLastModified = obj.user ?: 'Data services'
if (!dr.hasErrors()) {
DataResource.withTransaction {
Expand Down
2 changes: 1 addition & 1 deletion grails-app/services/au/org/ala/collectory/ExternalDataService.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -162,7 +162,7 @@ class ExternalDataService {
if (dr.hasErrors()) {
throw new ExternalResourceException("Created resource has errors", "manage.note.note02", resource.name, dr.errors)
}
dr.addExternalIdentifier(resource.guid, adaptor.source, resource.source)
externalIdentifierService.addExternalIdentifier(dr.uid, resource.guid, adaptor.source, resource.source)
resource.uid = dr.uid
resource.addNote("manage.note.note03", resource.uid)
}
Expand Down
10 changes: 5 additions & 5 deletions grails-app/views/manage/repatriate.gsp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,11 +30,11 @@
</div>
<div class="form-group">
<label for="country"><g:message code="manage.repatriationCountry.label06" /><cl:helpText code="manage.extload.label06.help"/></label>
<g:select name="country" class="form-control" from="${countryMap.entrySet()}" optionKey="key" optionValue="value" values="${configuration.country}" value="${params.country}"/>
<g:select name="country" class="form-control" from="${countryMap.entrySet()}" optionKey="key" optionValue="value" value="${configuration.country}"/>
</div>
<div class="form-group hide">
<label for="recordType"><g:message code="manage.extload.label07" /><cl:helpText code="manage.extload.label07.help"/></label>
<g:select name="recordType" class="form-control" from="${datasetTypeMap.entrySet()}" optionKey="key" optionValue="value" values="${configuration.recordType}"/>
<g:select name="recordType" class="form-control" from="${datasetTypeMap.entrySet()}" optionKey="key" optionValue="value" value="${configuration.recordType}"/>
</div>
<div class="form-group hide">
<label for="name"><g:message code="manage.extload.label01" /><cl:helpText code="manage.extload.label01.help"/></label>
Expand All @@ -57,17 +57,17 @@
</div>
<div class="form-inline">
<label for="maxNoOfDatasets"><g:message code="manage.extload.label13" /><cl:helpText code="manage.extload.label13.help"/></label>
<g:field type="number" name="maxNoOfDatasets" class="form-control form-control-sm" value="25" />
<g:field type="number" name="maxNoOfDatasets" class="form-control form-control-sm" value="${maxDatasetCount}" />
</div>
<br/>
<div class="form-inline">
<label for="minRecordCount"><g:message code="manage.extload.label11" /><cl:helpText code="manage.extload.label11.help"/></label>
<g:field type="number" name="minRecordCount" class="form-control form-control-sm" value="10000" />
<g:field type="number" name="minRecordCount" class="form-control form-control-sm" value="${minRecordCount}" />
</div>
<br/>
<div class="form-inline">
<label for="maxRecordCount"><g:message code="manage.extload.label12" /><cl:helpText code="manage.extload.label12.help"/></label>
<g:field type="number" name="maxRecordCount" class="form-control form-control-sm" value="1000000" />
<g:field type="number" name="maxRecordCount" class="form-control form-control-sm" value="${maxRecordCount}" />
</div>
<div>
<span class="button"><input type="submit" name="performReview" value="Review" class="save btn btn-default"></span>
Expand Down
17 changes: 10 additions & 7 deletions grails-app/views/manage/repatriateReview.gsp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
content="${createLink(action: 'list', controller: 'manage')},${message(code: 'manage.list.title01')}"
/>
<meta name="breadcrumbs"
content="${createLink(action: 'list', controller: 'manage')},Repatriation tools"
content="${createLink(action: 'repatriate', controller: 'manage')},Repatriation tools"
/>
<meta name="layout" content="${grailsApplication.config.skin.layout}" />
<title><g:message code="manage.extloadr.title" /></title>
Expand Down Expand Up @@ -49,11 +49,12 @@
<th>ID</th>
<th><g:message code="manage.extloadr.label01"/></th>
<th><g:message code="manage.extloadr.label03"/></th>
<th><g:message code="manage.extloadr.label10"/></th>
<th><g:message code="manage.extloadr.label04"/></th>
<th><g:message code="manage.extloadr.label05"/></th>
<th><g:message code="manage.extloadr.label06"/> <button type="btn btn-sm" onclick="invertColumn('.addResource'); return false"><span class="glyphicon glyphicon-check"></span></button></th>
<th><g:message code="manage.extloadr.label07"/> <button type="btn btn-sm" onclick="invertColumn('.updateMetadata'); return false"><span class="glyphicon glyphicon-check"></span></button></th>
<th><g:message code="manage.extloadr.label08"/> <button type="btn btn-sm" onclick="invertColumn('.updateConnection'); return false"><span class="glyphicon glyphicon-check"></span></button></th>
<th><g:message code="manage.extloadr.label06"/> <span class="btn btn-default btn-xs" onclick="invertColumn('.addResource'); return false">x</span></th>
<th><g:message code="manage.extloadr.label07"/> <span class="btn btn-default btn-xs" onclick="invertColumn('.updateMetadata'); return false">x</span></th>
<th><g:message code="manage.extloadr.label08"/> <span class="btn btn-default btn-xs" onclick="invertColumn('.updateConnection'); return false">x</span></th>
<th><g:message code="manage.extloadr.label09"/></th>
</tr>
</thead>
Expand All @@ -69,8 +70,9 @@
<g:hiddenField name="resources[${rs}].guid" value="${res.guid}"/>
<g:hiddenField name="resources[${rs}].source" value="${res.source}"/>
<g:hiddenField name="resources[${rs}].country" value="${configuration.country}"/>
<g:textField class="resource-name col-xs-4" name="resources[${rs}].name" value="${res.name}" />
<g:textField class="resource-name col-xs-4" style="width: 24em" name="resources[${rs}].name" value="${res.name}" />
</td>
<td>${res.type}</td>
<td><span title="<g:message code="manage.extstatus.${res.status}.detail"/>"><g:message code="manage.extstatus.${res.status}"/></span></td>
<td class="resource-mapping"><span id="existing-${rs}"><g:if test="${res.uid}">
<g:link controller="dataResource" action="show" id="${res.uid}" target="_new"> <g:fieldValue field="uid" bean="${res}"/></g:link>
Expand All @@ -80,7 +82,7 @@
<td><g:checkBox name="resources[${rs}].addResource" value="${res.addResource}"/></td>
<td><g:checkBox name="resources[${rs}].updateMetadata" value="${res.updateMetadata}"/></td>
<td><g:checkBox name="resources[${rs}].updateConnection" value="${res.updateConnection}"/></td>
<td>${res.recordCount}</td>
<td><g:formatNumber number="${res.recordCount}" format="###,###,##0" /></td>
</tr>
</g:each>
</g:if>
Expand Down Expand Up @@ -136,10 +138,11 @@
null,
null,
null,
null,
{"orderable": false},
{"orderable": false},
{"orderable": false},
{"orderable": false}
null
]
});
} );
Expand Down
2 changes: 1 addition & 1 deletion grails-app/views/public/_externalIdentifiers.gsp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
<h4><g:message code="public.sdr.externalIdentifiers.title" /></h4>
<ul>
<g:each var="ext" in="${external}">
<li><g:if test="${ext.uri}"><a href="${ext.uri}" class="external" target="_blank"><g:fieldValue bean="${ext}" field="label"/></a></g:if><g:else><g:fieldValue bean="${ext}" field="label"/></g:else></li>
<li><g:if test="${ext.validUri}"><a href="${ext.uri}" class="external" target="_blank"><g:fieldValue bean="${ext}" field="label"/></a></g:if><g:else><g:fieldValue bean="${ext}" field="label"/></g:else></li>
</g:each>
</ul>
</section>
Expand Down
8 changes: 7 additions & 1 deletion sbdi/data/config/collectory-config.properties
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -122,7 +122,7 @@ skin.taxaLinks.baseUrl = https://species.biodiversitydata.se/species/
spatial.baseURL = https://spatial.biodiversitydata.se/

# GBIF base URL for webservices
gbifApiUrl=http://api.gbif.org/v1
gbifApiUrl=https://api.gbif.org/v1/

# GBIF Registration enables the Collectory to create and update organisations and datasets
# This mode of operation is only expected to be used by National Nodes running the ALA as a publishing gateway to GBIF.
Expand Down Expand Up @@ -202,3 +202,9 @@ userdetails.api.url=https://auth.biodiversitydata.se/userdetails/
gbifDefaultEntityCountry=SWE

sitemap.dir=/data/collectory/data/sitemap

repatriate.defaults.country=SE
repatriate.defaults.dataProviderUId=dp4
repatriate.defaults.maxDatasetCount=5
repatriate.defaults.minRecordCount=100
repatriate.defaults.maxRecordCount=5000000
26 changes: 15 additions & 11 deletions src/main/groovy/au/org/ala/collectory/resources/gbif/GbifDataSourceAdapter.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ class GbifDataSourceAdapter extends DataSourceAdapter {
static final MessageFormat DATASET_GET = new MessageFormat("dataset/{0}")
static final MessageFormat DATASET_RECORD_COUNT = new MessageFormat("occurrence/count?datasetKey={0}")
static final MessageFormat DOWNLOAD_STATUS = new MessageFormat("occurrence/download/{0}")
static final DateFormat TIMESTAMP_FORMAT= new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSZ")
static final DateFormat TIMESTAMP_FORMAT= new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSXXX")

static LICENSE_MAP = [
"https://creativecommons.org/publicdomain/zero/1.0/legalcode": [licenseType: "CC0", licenseVersion: "1.0" ],
Expand All @@ -44,19 +44,19 @@ class GbifDataSourceAdapter extends DataSourceAdapter {
"http://creativecommons.org/licenses/by/4.0/legalcode": [licenseType: "CC BY", licenseVersion: "4.0" ]
]
static TYPE_MAP = [
"CHECKLIST" : "species-list",
"METADATA" : "document",
"OCCURRENCE" : "records",
"SAPLING_EVENT": "records"
"CHECKLIST" : "species-list",
"METADATA" : "document",
"OCCURRENCE" : "records",
"SAMPLING_EVENT": "records"
]
static DATASET_TYPES = [
"OCCURRENCE" : "Occurrence Records" // We only allow occurrence records at the moment
]
static CONTENT_MAP = [
"CHECKLIST" : ["species list", "taxonomy", "gbif import"],
"METADATA" : ["gbif import"],
"OCCURRENCE" : ["point occurrence data", "gbif import"],
"SAPLING_EVENT": ["point occurrence data", "gbif import"]
"CHECKLIST" : ["species list", "taxonomy", "gbif import"],
"METADATA" : ["gbif import"],
"OCCURRENCE" : ["point occurrence data", "gbif import"],
"SAMPLING_EVENT": ["point occurrence data", "gbif import"]
]
static DOWNLOAD_STATUS_MAP = [
"CANCELLED" : TaskPhase.CANCELLED,
Expand Down Expand Up @@ -142,7 +142,8 @@ class GbifDataSourceAdapter extends DataSourceAdapter {

@Override
ExternalResourceBean createExternalResource(Map external) {
ExternalResourceBean ext = new ExternalResourceBean(name: external.name, guid: external.guid, source: external.source, sourceUpdated: external.dataCurrency)
ExternalResourceBean ext = new ExternalResourceBean(name: external.name, guid: external.guid,
source: external.source, sourceUpdated: external.dataCurrency, type: external.resourceType)
DataResource dr = ext.resolve(source)
if (!dr) {
ext.status = ExternalResourceBean.ResourceStatus.NEW
Expand Down Expand Up @@ -205,7 +206,10 @@ class GbifDataSourceAdapter extends DataSourceAdapter {
address: address,
phone: phone,
email: email,
pubDescription: dataset.description,
pubDescription: dataset.description
.replaceAll("(?i)<p[^>]*>", "\n")
.replaceAll("(?i)</p>", "")
.replaceAll("(?i)<br\\s*/?>", "\n"),
state: address?.state,
websiteUrl: dataset.homepage,
rights: dataset.rights,
Expand Down
15 changes: 13 additions & 2 deletions src/main/groovy/au/org/ala/collectory/resources/gbif/GbifRepatDataSourceAdapter.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,16 +17,24 @@ class GbifRepatDataSourceAdapter extends GbifDataSourceAdapter {
static final SOURCE = "GBIF_REPATRIATION"
GbifService gbifService

static final String OCCURRENCE_REPAT_SEARCH = "occurrence/search?repatriated=true&country={0}&type={1}&offset=0&limit=0&facet=datasetKey&facetLimit=10000"
static final String OCCURRENCE_REPAT_SEARCH = "occurrence/search?country={0}&type={1}&offset=0&limit=0&facet=datasetKey&facetLimit=10000"
static final String COUNTRY_DATASETS = "dataset?country={0}&limit=10000"

GbifRepatDataSourceAdapter(DataSourceConfiguration configuration) {
super(configuration)
}

List<String> fetchCountryDatasetKeys() {
String url = MessageFormat.format(COUNTRY_DATASETS, configuration.country)
JSONObject json = getJSONWS(url, false)
json.results.collect{ it -> it.key }
}

@Override
List<Map> datasets() throws ExternalResourceException {
def keys = []
def datasets = []
def countryDatasetKeys = fetchCountryDatasetKeys()

LOGGER.info("Requesting dataset lists configuration.country: ${configuration.country}")
String url = MessageFormat.format(OCCURRENCE_REPAT_SEARCH, configuration.country, configuration.recordType)
Expand All @@ -35,7 +43,10 @@ class GbifRepatDataSourceAdapter extends GbifDataSourceAdapter {
json.facets[0].counts.each {
keys << it.name

if (it.count >= configuration.minRecordCount && it.count <= configuration.maxRecordCount && datasets.size() < configuration.maxNoOfDatasets) {
if (!countryDatasetKeys.contains(it.name) &&
it.count >= configuration.minRecordCount &&
it.count <= configuration.maxRecordCount &&
datasets.size() < configuration.maxNoOfDatasets) {
LOGGER.info("Getting metadata for ${it.name} = ${it.count}")
def dataset = getDataset(it.name, it.count)
if (dataset.name) {
Expand Down