Release 4.4.0

#273 fix deletion of a list which has a large mount of KVP records
fix bugs on params and optimise the list-item query

.gitignore

@@ -6,3 +6,12 @@ out
 /.slcache
 /logs
 *.iml
+grails-app/.DS_Store
+grails-app/controllers/.DS_Store
+grails-app/controllers/au/.DS_Store
+grails-app/controllers/au/org/.DS_Store
+grails-app/controllers/au/org/ala/.DS_Store
+grails-app/services/.DS_Store
+grails-app/services/au/.DS_Store
+grails-app/services/au/org/.DS_Store
+grails-app/services/au/org/ala/.DS_Store

build.gradle

@@ -16,7 +16,9 @@ plugins {
  id "com.gorylenko.gradle-git-properties" version "2.4.1"
 }
 
-version "4.1.3"
+
+version "4.4.0"
+
 group "au.org.ala"
 
 apply plugin:"eclipse"

grails-app/conf/application.yml

@@ -1,4 +1,8 @@
 ---
+
+logging:
+ config: /data/specieslist-webapp/config/logback.xml
+
 grails:
  profile: web
  codegen:
@@ -135,7 +139,7 @@ dataSource:
 environments:
  development:
  grails:
- serverURL: "http://dev.ala.org.au:8080"
+ serverURL: "http://localhost:8080"
  dataSource:
  dbCreate: none
  url: jdbc:mysql://localhost/specieslist?autoReconnect=true&connectTimeout=0&useUnicode=true&characterEncoding=UTF-8

grails-app/conf/logback.xml

@@ -3,6 +3,9 @@
 
  <conversionRule conversionWord="clr" converterClass="org.springframework.boot.logging.logback.ColorConverter" />
  <conversionRule conversionWord="wex" converterClass="org.springframework.boot.logging.logback.WhitespaceThrowableProxyConverter" />
+ <logger name="org.hibernate" level="ERROR"/>
+ <logger name="org.grails.config" level="ERROR"/>
+
 
  <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
  <encoder>
@@ -11,7 +14,7 @@
  </encoder>
  </appender>
 
- <root level="warn">
+ <root level="INFO">
  <appender-ref ref="STDOUT" />
  </root>
 </configuration>

grails-app/controllers/au/org/ala/specieslist/SpeciesListController.groovy

@@ -83,7 +83,9 @@ class SpeciesListController {
  def sl = SpeciesList.get(params.id)
  if(sl){
  helperService.deleteDataResourceForList(sl.dataResourceUid)
- sl.delete(flush: true)
+ SpeciesListItem.executeUpdate("delete from SpeciesListItem where dataResourceUid = :dataResourceUid", ["dataResourceUid": sl.dataResourceUid])
+ SpeciesListKVP.executeUpdate("delete from SpeciesListKVP where dataResourceUid = :dataResourceUid", ["dataResourceUid": sl.dataResourceUid])
+ SpeciesList.executeUpdate("delete from SpeciesList where dataResourceUid = :dataResourceUid", ["dataResourceUid": sl.dataResourceUid])
  }
  redirect(action: 'list')
  }

grails-app/controllers/au/org/ala/specieslist/WebServiceController.groovy

@@ -187,42 +187,35 @@ class WebServiceController {
  def getListItemsForSpecies() {
  def guid = params.guid.replaceFirst("https:/", "https://")
  def lists = params.dr?.split(",")
- def isBIE = params.boolean('isBIE')
- def props = [fetch: [kvpValues: 'join', mylist: 'join']]
-
- def results = queryService.getFilterListItemResult(props, params, guid, lists, null)
+ def queryParams = params.subMap(["max","sort","offset"])
 
+ def results = queryService.getListForSpecies(guid, params.isBIE, lists, queryParams )
  log.debug("RESULTS: " + results)
 
  // fetch lists that this user has access to view
  def hidePrivateLists = grailsApplication.config.getProperty('publicview.hidePrivateLists', Boolean, false)
  def permittedPrivateLists = queryService.visibleLists(false, hidePrivateLists)
 
- def filteredRecords = results.findAll { !it.mylist.isPrivate || permittedPrivateLists.contains(it.dataResourceUid) }
-
- if (isBIE) {
- // BIE only want lists with isBIE == true
- filteredRecords = filteredRecords.findAll { it.mylist.isBIE }
- }
-
- def listOfRecordMaps = filteredRecords.collect { li -> // don't output private lists
+ def listOfRecordMaps = results.findResults {
+ // don't output private lists
+ (!it.mylist.isPrivate) || permittedPrivateLists.contains(it.dataResourceUid) ?
  [
- dataResourceUid: li.dataResourceUid,
- guid : li.guid,
+ dataResourceUid: it.dataResourceUid,
+ guid : it.guid,
  list : [
- username: li.mylist.username,
- listName: li.mylist.listName,
- sds : li.mylist.isSDS ?: false,
- isBIE : li.mylist.isBIE ?: false
+ username: it.mylist.username,
+ listName: it.mylist.listName,
+ sds : it.mylist.isSDS ?: false,
+ isBIE : it.mylist.isBIE ?: false
  ],
- kvpValues : li.kvpValues.collect { kvp ->
+ kvpValues : it.kvpValues.collect { kvp ->
  [
  key : kvp.key,
  value : kvp.value,
  vocabValue: kvp.vocabValue
  ]
  }
- ]
+ ] : null
  }
  render listOfRecordMaps as JSON
  }
@@ -1337,5 +1330,10 @@ class WebServiceController {
  return isAllowed
  }
 
+ def handleException(final Exception e ) {
+ log.error(e.message)
+ return {error: e.message}
+ }
+
 
 }

grails-app/services/au/org/ala/specieslist/HelperService.groovy

@@ -28,6 +28,7 @@ import org.grails.web.json.JSONArray
 import org.nibor.autolink.*
 import org.springframework.context.MessageSource
 import org.springframework.context.i18n.LocaleContextHolder
+import org.apache.http.util.EntityUtils
 
 import javax.annotation.PostConstruct
 
@@ -112,7 +113,8 @@ class HelperService {
  requestContentType = ContentType.JSON
  headers."Authorization" = "${grailsApplication.config.registryApiKey}"
  response.success = { resp ->
- log.info(resp?.toString())
+ def result = (resp.getEntity() != null ? EntityUtils.toString(resp.getEntity()) : "")
+ log.info("${drId} has been deleted from ${grailsApplication.config.collectory.baseURL} with ${result}")
  }
  response.failure = { resp ->
  log.error("Delete request for ${drId} failed with status ${resp.status}")

grails-app/services/au/org/ala/specieslist/QueryService.groovy

@@ -15,7 +15,7 @@
 
 package au.org.ala.specieslist
 
-
+import groovy.time.*
 import org.hibernate.Criteria
 import org.hibernate.criterion.CriteriaQuery
 import org.hibernate.criterion.Order
@@ -180,9 +180,11 @@ class QueryService {
  if (query){
  selectedFacets << [query: key, facet: query]
  } else if (key == LIST_TYPE){
- def cleanedVaue = value.replaceAll("eq:", "")
- query = listTyoeFacets.get(cleanedVaue)
- selectedFacets << [query: key, facet: query]
+ if (value) {
+ def cleanedValue = value.replaceAll("eq:", "")
+ query = listTyoeFacets.get(cleanedValue)
+ selectedFacets << [query: key, facet: query]
+ }
  } else if (key == WKT){
  query = WKT_QUERY
  selectedFacets << [query: WKT, facet: [label:'spatialBounds.list.label']]
@@ -445,8 +447,38 @@ class QueryService {
  order = order ?: ASC
  c.order(new Order(sort, ASC.equalsIgnoreCase(order)))
  }
+ /*
+ * Retrieves the species list items by given guid.
+ *
+ * @param queryParams : only supports: max, offset, sort
+ * @param guid
+ * @param isBIE
+ * @param lists: data resource ids
+ * @return
+ */
+ def getListForSpecies(guid, isBIE, lists, queryParams ) {
+ def speciesListProperties = getSpeciesListProperties()
+ def c = SpeciesListItem.createCriteria()
+
+ def results = c.list(queryParams) {
+ eq(GUID, guid)
+ if (isBIE) {
+ mylist {
+ eq("isBIE", isBIE.toBoolean())
+ }
+ }
+ if (lists) {
+ 'in'(DATA_RESOURCE_UID, lists)
+ }
+ }
+ return results
+ }
 
  /**
+ * @Todo
+ * Fix bug when param: max is set
+ * @See issue #271
+ *
  * retrieves the species list items that obey the supplied filters.
  *
  * When a distinct field is provided the values of the field are returned rather than a SpeciesListItem
@@ -461,6 +493,9 @@ class QueryService {
  def speciesListProperties = getSpeciesListProperties()
  def c = SpeciesListItem.createCriteria()
 
+ //params – pagination parameters (max, offset, etc...) closure – The closure to execute
+ //sort, max, offset
+ // max parameter does not work, @see #271
  c.list(props += params) {
  //set the results transformer so that we don't get duplicate records because of
  // the 1:many relationship between a list item and KVP
@@ -513,6 +548,7 @@ class QueryService {
  }
  }
  }
+
  }
  }
 
@@ -793,12 +829,14 @@ class QueryService {
  queryParameters.qCommonName = '%'+q+'%'
  queryParameters.qRawScientificName = '%'+q+'%'
  }
-
+ def timeStart = new Date()
  def results = SpeciesListItem.executeQuery("select kvp.key, kvp.value, kvp.vocabValue, count(sli) as cnt from SpeciesListItem as sli " +
  "join sli.kvpValues as kvp where sli.dataResourceUid = :druid ${ids ? 'and sli.id in (:ids)' : ''} " +
  "${q ? 'and (sli.matchedName like :qMatchedName or sli.commonName like :qCommonName or sli.rawScientificName like :qRawScientificName) ' : ''} " +
  "group by kvp.key, kvp.value, kvp.vocabValue, kvp.itemOrder, kvp.key order by kvp.itemOrder, kvp.key, cnt desc",
  queryParameters)
+ def timeStop = new Date()
+ log.info("Query KVP of " + fqs + "took " + TimeCategory.minus(timeStop, timeStart))
 
  //obtain the families from the common list facets
  def commonResults = SpeciesListItem.executeQuery("select sli.family, count(sli) as cnt from SpeciesListItem sli " +
@@ -816,15 +854,15 @@ class QueryService {
  } else {
  def qParam = '%'+q+'%'
  def queryParameters = q ? [dataResourceUid: id, matchedName: qParam, commonName: qParam, rawScientificName: qParam] : [dataResourceUid: id]
-
+ def timeStart = new Date()
  def results = SpeciesListItem.executeQuery('select kvp.key, kvp.value, kvp.vocabValue, count(sli) as cnt from SpeciesListItem as sli ' +
  'join sli.kvpValues as kvp where sli.dataResourceUid = :dataResourceUid ' +
  "${q ? 'and (sli.matchedName like :matchedName or sli.commonName like :commonName or sli.rawScientificName like :rawScientificName) ' : ''} " +
  'group by kvp.key, kvp.value, kvp.vocabValue, kvp.itemOrder order by kvp.itemOrder, kvp.key, cnt desc',
  queryParameters)
-
+ def timeStop = new Date()
+ log.info("Query KVP of " + id + "took " + TimeCategory.minus(timeStop, timeStart))
  properties = results.findAll{it[1].length()<maxLengthForFacet}.groupBy{it[0]}.findAll{it.value.size()>1 }
-
  //obtain the families from the common list facets
  def commonResults = SpeciesListItem.executeQuery('select family, count(*) as cnt from SpeciesListItem ' +
  'where family is not null AND dataResourceUid = :dataResourceUid ' +

grails-app/views/_speciesList.gsp

@@ -48,8 +48,6 @@
  var url = "${request.contextPath}"+"/speciesList/"+action+ "/"+listId;
  //console.log("Dialog ACTION ITEMS",listId, url)
  $.post(url, function(data){
- //alert('Value returned from service: ' + data.uid);
- alert('${message(code:'admin.lists.actions.button.message.ok', default:'Action was successful!')}');
  window.location.reload()
  }).error(function(jqXHR, textStatus, error) {
  alert("An error occurred: " + error + " - " + jqXHR.responseText);

grails-app/views/speciesListItem/list.gsp

@@ -225,7 +225,7 @@
  var url = "${request.contextPath}"+"/speciesList/delete/" + listId;
 
  $.post(url, function(data){
- window.location.reload()
+ window.location.replace("${request.contextPath}"+"/speciesList/list");
  }).error(function(jqXHR, textStatus, error) {
  alert("An error occurred: " + error + " - " + jqXHR.responseText);
  $(modal).modal('hide');